From 846a6d8717b6a8e0c125f79f049734f87ec07010 Mon Sep 17 00:00:00 2001 From: Ilya Fedin Date: Mon, 18 Mar 2024 17:57:44 +0400 Subject: [PATCH] Update scudo to 18.1.1 --- Telegram/ThirdParty/scudo/CMakeLists.txt | 13 +- Telegram/ThirdParty/scudo/allocator_common.h | 85 +++ Telegram/ThirdParty/scudo/allocator_config.h | 53 +- Telegram/ThirdParty/scudo/atomic_helpers.h | 8 +- .../scudo/benchmarks/malloc_benchmark.cpp | 4 - Telegram/ThirdParty/scudo/chunk.h | 13 - Telegram/ThirdParty/scudo/combined.h | 230 ++++---- Telegram/ThirdParty/scudo/common.cpp | 18 - Telegram/ThirdParty/scudo/common.h | 38 +- .../ThirdParty/scudo/condition_variable.h | 60 ++ .../scudo/condition_variable_base.h | 56 ++ .../scudo/condition_variable_linux.cpp | 52 ++ .../scudo/condition_variable_linux.h | 38 ++ Telegram/ThirdParty/scudo/flags.cpp | 3 + Telegram/ThirdParty/scudo/flags.inc | 12 +- Telegram/ThirdParty/scudo/flags_parser.cpp | 30 +- Telegram/ThirdParty/scudo/flags_parser.h | 3 +- .../scudo/fuzz/get_error_info_fuzzer.cpp | 9 +- .../scudo/include/scudo/interface.h | 17 +- Telegram/ThirdParty/scudo/linux.cpp | 50 +- Telegram/ThirdParty/scudo/local_cache.h | 125 +--- Telegram/ThirdParty/scudo/mem_map.h | 5 +- Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp | 12 +- Telegram/ThirdParty/scudo/mem_map_linux.cpp | 153 +++++ Telegram/ThirdParty/scudo/mem_map_linux.h | 67 +++ Telegram/ThirdParty/scudo/mutex.h | 19 +- Telegram/ThirdParty/scudo/options.h | 2 +- Telegram/ThirdParty/scudo/platform.h | 14 + Telegram/ThirdParty/scudo/primary32.h | 298 +++++++--- Telegram/ThirdParty/scudo/primary64.h | 551 +++++++++++------- Telegram/ThirdParty/scudo/release.cpp | 3 +- Telegram/ThirdParty/scudo/release.h | 171 +++--- Telegram/ThirdParty/scudo/report.cpp | 33 +- Telegram/ThirdParty/scudo/report.h | 8 +- Telegram/ThirdParty/scudo/report_linux.cpp | 58 ++ Telegram/ThirdParty/scudo/report_linux.h | 34 ++ .../ThirdParty/scudo/rss_limit_checker.cpp | 37 -- Telegram/ThirdParty/scudo/rss_limit_checker.h | 63 -- Telegram/ThirdParty/scudo/secondary.h | 186 ++++-- Telegram/ThirdParty/scudo/size_class_map.h | 26 +- Telegram/ThirdParty/scudo/stack_depot.h | 5 +- .../ThirdParty/scudo/tests/CMakeLists.txt | 15 +- .../ThirdParty/scudo/tests/chunk_test.cpp | 23 - .../ThirdParty/scudo/tests/combined_test.cpp | 188 ++++-- .../ThirdParty/scudo/tests/common_test.cpp | 23 - .../scudo/tests/condition_variable_test.cpp | 59 ++ .../ThirdParty/scudo/tests/memtag_test.cpp | 10 +- .../ThirdParty/scudo/tests/primary_test.cpp | 43 +- .../ThirdParty/scudo/tests/release_test.cpp | 42 +- .../ThirdParty/scudo/tests/report_test.cpp | 1 - .../scudo/tests/scudo_hooks_test.cpp | 114 ---- .../ThirdParty/scudo/tests/scudo_unit_test.h | 6 + .../scudo/tests/scudo_unit_test_main.cpp | 4 +- .../scudo/tests/size_class_map_test.cpp | 4 - Telegram/ThirdParty/scudo/tests/tsd_test.cpp | 4 + .../scudo/tests/wrappers_c_test.cpp | 201 ++++++- .../scudo/tests/wrappers_cpp_test.cpp | 143 +++-- Telegram/ThirdParty/scudo/trusty.cpp | 6 +- Telegram/ThirdParty/scudo/tsd.h | 17 +- Telegram/ThirdParty/scudo/tsd_shared.h | 5 + Telegram/ThirdParty/scudo/vector.h | 51 +- Telegram/ThirdParty/scudo/wrappers_c.cpp | 3 + Telegram/ThirdParty/scudo/wrappers_c.inc | 108 +++- .../ThirdParty/scudo/wrappers_c_bionic.cpp | 21 +- Telegram/ThirdParty/scudo/wrappers_cpp.cpp | 66 ++- 65 files changed, 2424 insertions(+), 1395 deletions(-) create mode 100644 Telegram/ThirdParty/scudo/allocator_common.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable_base.h create mode 100644 Telegram/ThirdParty/scudo/condition_variable_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/condition_variable_linux.h create mode 100644 Telegram/ThirdParty/scudo/mem_map_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/mem_map_linux.h create mode 100644 Telegram/ThirdParty/scudo/report_linux.cpp create mode 100644 Telegram/ThirdParty/scudo/report_linux.h delete mode 100644 Telegram/ThirdParty/scudo/rss_limit_checker.cpp delete mode 100644 Telegram/ThirdParty/scudo/rss_limit_checker.h create mode 100644 Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp delete mode 100644 Telegram/ThirdParty/scudo/tests/scudo_hooks_test.cpp diff --git a/Telegram/ThirdParty/scudo/CMakeLists.txt b/Telegram/ThirdParty/scudo/CMakeLists.txt index 094c23a73..60092005c 100644 --- a/Telegram/ThirdParty/scudo/CMakeLists.txt +++ b/Telegram/ThirdParty/scudo/CMakeLists.txt @@ -25,7 +25,7 @@ append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic SCUDO_CFLAGS) append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto SCUDO_CFLAGS) if(COMPILER_RT_DEBUG) - list(APPEND SCUDO_CFLAGS -O0 -DSCUDO_DEBUG=1) + list(APPEND SCUDO_CFLAGS -O0 -DSCUDO_DEBUG=1 -DSCUDO_ENABLE_HOOKS=1) else() list(APPEND SCUDO_CFLAGS -O3) endif() @@ -56,11 +56,15 @@ if(ANDROID) endif() set(SCUDO_HEADERS + allocator_common.h allocator_config.h atomic_helpers.h bytemap.h checksum.h chunk.h + condition_variable.h + condition_variable_base.h + condition_variable_linux.h combined.h common.h flags_parser.h @@ -74,6 +78,7 @@ set(SCUDO_HEADERS mem_map.h mem_map_base.h mem_map_fuchsia.h + mem_map_linux.h mutex.h options.h platform.h @@ -82,7 +87,7 @@ set(SCUDO_HEADERS quarantine.h release.h report.h - rss_limit_checker.h + report_linux.h secondary.h size_class_map.h stack_depot.h @@ -102,6 +107,7 @@ set(SCUDO_HEADERS set(SCUDO_SOURCES checksum.cpp common.cpp + condition_variable_linux.cpp crc32_hw.cpp flags_parser.cpp flags.cpp @@ -109,9 +115,10 @@ set(SCUDO_SOURCES linux.cpp mem_map.cpp mem_map_fuchsia.cpp + mem_map_linux.cpp release.cpp report.cpp - rss_limit_checker.cpp + report_linux.cpp string_utils.cpp timing.cpp ) diff --git a/Telegram/ThirdParty/scudo/allocator_common.h b/Telegram/ThirdParty/scudo/allocator_common.h new file mode 100644 index 000000000..95f4776ac --- /dev/null +++ b/Telegram/ThirdParty/scudo/allocator_common.h @@ -0,0 +1,85 @@ +//===-- allocator_common.h --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_ALLOCATOR_COMMON_H_ +#define SCUDO_ALLOCATOR_COMMON_H_ + +#include "common.h" +#include "list.h" + +namespace scudo { + +template struct TransferBatch { + typedef typename SizeClassAllocator::SizeClassMap SizeClassMap; + typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; + + static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint; + void setFromArray(CompactPtrT *Array, u16 N) { + DCHECK_LE(N, MaxNumCached); + Count = N; + memcpy(Batch, Array, sizeof(Batch[0]) * Count); + } + void appendFromArray(CompactPtrT *Array, u16 N) { + DCHECK_LE(N, MaxNumCached - Count); + memcpy(Batch + Count, Array, sizeof(Batch[0]) * N); + // u16 will be promoted to int by arithmetic type conversion. + Count = static_cast(Count + N); + } + void appendFromTransferBatch(TransferBatch *B, u16 N) { + DCHECK_LE(N, MaxNumCached - Count); + DCHECK_GE(B->Count, N); + // Append from the back of `B`. + memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N); + // u16 will be promoted to int by arithmetic type conversion. + Count = static_cast(Count + N); + B->Count = static_cast(B->Count - N); + } + void clear() { Count = 0; } + void add(CompactPtrT P) { + DCHECK_LT(Count, MaxNumCached); + Batch[Count++] = P; + } + void moveToArray(CompactPtrT *Array) { + memcpy(Array, Batch, sizeof(Batch[0]) * Count); + clear(); + } + u16 getCount() const { return Count; } + bool isEmpty() const { return Count == 0U; } + CompactPtrT get(u16 I) const { + DCHECK_LE(I, Count); + return Batch[I]; + } + TransferBatch *Next; + +private: + CompactPtrT Batch[MaxNumCached]; + u16 Count; +}; + +// A BatchGroup is used to collect blocks. Each group has a group id to +// identify the group kind of contained blocks. +template struct BatchGroup { + // `Next` is used by IntrusiveList. + BatchGroup *Next; + // The compact base address of each group + uptr CompactPtrGroupBase; + // Cache value of SizeClassAllocatorLocalCache::getMaxCached() + u16 MaxCachedPerBatch; + // Number of blocks pushed into this group. This is an increment-only + // counter. + uptr PushedBlocks; + // This is used to track how many bytes are not in-use since last time we + // tried to release pages. + uptr BytesInBGAtLastCheckpoint; + // Blocks are managed by TransferBatch in a list. + SinglyLinkedList> Batches; +}; + +} // namespace scudo + +#endif // SCUDO_ALLOCATOR_COMMON_H_ diff --git a/Telegram/ThirdParty/scudo/allocator_config.h b/Telegram/ThirdParty/scudo/allocator_config.h index 315a04f76..3c6aa3acb 100644 --- a/Telegram/ThirdParty/scudo/allocator_config.h +++ b/Telegram/ThirdParty/scudo/allocator_config.h @@ -11,6 +11,7 @@ #include "combined.h" #include "common.h" +#include "condition_variable.h" #include "flags.h" #include "primary32.h" #include "primary64.h" @@ -82,6 +83,14 @@ namespace scudo { // // Defines the minimal & maximal release interval that can be set. // static const s32 MinReleaseToOsIntervalMs = INT32_MIN; // static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; +// +// // Use condition variable to shorten the waiting time of refillment of +// // freelist. Note that this depends on the implementation of condition +// // variable on each platform and the performance may vary so that it +// // doesn't guarantee a performance benefit. +// // Note that both variables have to be defined to enable it. +// static const bool UseConditionVariable = true; +// using ConditionVariableT = ConditionVariableLinux; // }; // // Defines the type of Primary allocator to use. // template using PrimaryT = SizeClassAllocator64; @@ -195,50 +204,6 @@ struct AndroidConfig { template using SecondaryT = MapAllocator; }; -struct AndroidSvelteConfig { - static const bool MaySupportMemoryTagging = false; - template - using TSDRegistryT = TSDRegistrySharedT; // Shared, max 2 TSDs. - - struct Primary { - using SizeClassMap = SvelteSizeClassMap; -#if SCUDO_CAN_USE_PRIMARY64 - static const uptr RegionSizeLog = 27U; - typedef u32 CompactPtrT; - static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const uptr GroupSizeLog = 18U; - static const bool EnableRandomOffset = true; - static const uptr MapSizeIncrement = 1UL << 18; -#else - static const uptr RegionSizeLog = 16U; - static const uptr GroupSizeLog = 16U; - typedef uptr CompactPtrT; -#endif - static const s32 MinReleaseToOsIntervalMs = 1000; - static const s32 MaxReleaseToOsIntervalMs = 1000; - }; - -#if SCUDO_CAN_USE_PRIMARY64 - template using PrimaryT = SizeClassAllocator64; -#else - template using PrimaryT = SizeClassAllocator32; -#endif - - struct Secondary { - struct Cache { - static const u32 EntriesArraySize = 16U; - static const u32 QuarantineSize = 32U; - static const u32 DefaultMaxEntriesCount = 4U; - static const uptr DefaultMaxEntrySize = 1UL << 18; - static const s32 MinReleaseToOsIntervalMs = 0; - static const s32 MaxReleaseToOsIntervalMs = 0; - }; - template using CacheT = MapAllocatorCache; - }; - - template using SecondaryT = MapAllocator; -}; - #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { static const bool MaySupportMemoryTagging = false; diff --git a/Telegram/ThirdParty/scudo/atomic_helpers.h b/Telegram/ThirdParty/scudo/atomic_helpers.h index d88f5d7be..a68ffd162 100644 --- a/Telegram/ThirdParty/scudo/atomic_helpers.h +++ b/Telegram/ThirdParty/scudo/atomic_helpers.h @@ -133,10 +133,10 @@ inline void atomic_store_relaxed(volatile T *A, typename T::Type V) { } template -inline typename T::Type atomic_compare_exchange(volatile T *A, - typename T::Type Cmp, - typename T::Type Xchg) { - atomic_compare_exchange_strong(A, &Cmp, Xchg, memory_order_acquire); +inline typename T::Type +atomic_compare_exchange_strong(volatile T *A, typename T::Type Cmp, + typename T::Type Xchg, memory_order MO) { + atomic_compare_exchange_strong(A, &Cmp, Xchg, MO); return Cmp; } diff --git a/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp b/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp index 2adec88da..4fb05b761 100644 --- a/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp +++ b/Telegram/ThirdParty/scudo/benchmarks/malloc_benchmark.cpp @@ -52,8 +52,6 @@ static const size_t MaxSize = 128 * 1024; // cleanly. BENCHMARK_TEMPLATE(BM_malloc_free, scudo::AndroidConfig) ->Range(MinSize, MaxSize); -BENCHMARK_TEMPLATE(BM_malloc_free, scudo::AndroidSvelteConfig) - ->Range(MinSize, MaxSize); #if SCUDO_CAN_USE_PRIMARY64 BENCHMARK_TEMPLATE(BM_malloc_free, scudo::FuchsiaConfig) ->Range(MinSize, MaxSize); @@ -99,8 +97,6 @@ static const size_t MaxIters = 32 * 1024; // cleanly. BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::AndroidConfig) ->Range(MinIters, MaxIters); -BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::AndroidSvelteConfig) - ->Range(MinIters, MaxIters); #if SCUDO_CAN_USE_PRIMARY64 BENCHMARK_TEMPLATE(BM_malloc_free_loop, scudo::FuchsiaConfig) ->Range(MinIters, MaxIters); diff --git a/Telegram/ThirdParty/scudo/chunk.h b/Telegram/ThirdParty/scudo/chunk.h index 32874a8df..9228df047 100644 --- a/Telegram/ThirdParty/scudo/chunk.h +++ b/Telegram/ThirdParty/scudo/chunk.h @@ -128,19 +128,6 @@ inline void loadHeader(u32 Cookie, const void *Ptr, reportHeaderCorruption(const_cast(Ptr)); } -inline void compareExchangeHeader(u32 Cookie, void *Ptr, - UnpackedHeader *NewUnpackedHeader, - UnpackedHeader *OldUnpackedHeader) { - NewUnpackedHeader->Checksum = - computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); - PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); - PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); - if (UNLIKELY(!atomic_compare_exchange_strong( - getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, - memory_order_relaxed))) - reportHeaderRace(Ptr); -} - inline bool isValid(u32 Cookie, const void *Ptr, UnpackedHeader *NewUnpackedHeader) { PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); diff --git a/Telegram/ThirdParty/scudo/combined.h b/Telegram/ThirdParty/scudo/combined.h index b17acc71f..4624f83d1 100644 --- a/Telegram/ThirdParty/scudo/combined.h +++ b/Telegram/ThirdParty/scudo/combined.h @@ -14,11 +14,11 @@ #include "flags.h" #include "flags_parser.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "quarantine.h" #include "report.h" -#include "rss_limit_checker.h" #include "secondary.h" #include "stack_depot.h" #include "string_utils.h" @@ -68,14 +68,13 @@ public: if (UNLIKELY(Header.State != Chunk::State::Quarantined)) reportInvalidChunkState(AllocatorAction::Recycling, Ptr); - Chunk::UnpackedHeader NewHeader = Header; - NewHeader.State = Chunk::State::Available; - Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + Header.State = Chunk::State::Available; + Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); - void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); - Cache.deallocate(NewHeader.ClassId, BlockBegin); + void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header); + Cache.deallocate(Header.ClassId, BlockBegin); } // We take a shortcut when allocating a quarantine batch by working with the @@ -118,9 +117,8 @@ public: DCHECK_EQ(Header.Offset, 0); DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch)); - Chunk::UnpackedHeader NewHeader = Header; - NewHeader.State = Chunk::State::Available; - Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); + Header.State = Chunk::State::Available; + Chunk::storeHeader(Allocator.Cookie, Ptr, &Header); Cache.deallocate(QuarantineClassId, reinterpret_cast(reinterpret_cast(Ptr) - Chunk::getHeaderSize())); @@ -149,9 +147,6 @@ public: initFlags(); reportUnrecognizedFlags(); - RssChecker.init(scudo::getFlags()->soft_rss_limit_mb, - scudo::getFlags()->hard_rss_limit_mb); - // Store some flags locally. if (getFlags()->may_return_null) Primary.Options.set(OptionBit::MayReturnNull); @@ -251,12 +246,14 @@ public: // - unlinking the local stats from the global ones (destroying the cache does // the last two items). void commitBack(TSD *TSD) { + TSD->assertLocked(/*BypassCheck=*/true); Quarantine.drain(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache())); TSD->getCache().destroy(&Stats); } void drainCache(TSD *TSD) { + TSD->assertLocked(/*BypassCheck=*/true); Quarantine.drainAndRecycle(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache())); TSD->getCache().drain(); @@ -299,7 +296,7 @@ public: #endif } - uptr computeOddEvenMaskForPointerMaybe(Options Options, uptr Ptr, + uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr, uptr ClassId) { if (!Options.get(OptionBit::UseOddEvenTags)) return 0; @@ -329,8 +326,6 @@ public: #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.shouldSample())) { if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { - if (UNLIKELY(&__scudo_allocate_hook)) - __scudo_allocate_hook(Ptr, Size); Stats.lock(); Stats.add(StatAllocated, GuardedAllocSlotSize); Stats.sub(StatFree, GuardedAllocSlotSize); @@ -363,19 +358,6 @@ public: } DCHECK_LE(Size, NeededSize); - switch (RssChecker.getRssLimitExceeded()) { - case RssLimitChecker::Neither: - break; - case RssLimitChecker::Soft: - if (Options.get(OptionBit::MayReturnNull)) - return nullptr; - reportSoftRSSLimit(RssChecker.getSoftRssLimit()); - break; - case RssLimitChecker::Hard: - reportHardRSSLimit(RssChecker.getHardRssLimit()); - break; - } - void *Block = nullptr; uptr ClassId = 0; uptr SecondaryBlockEnd = 0; @@ -384,11 +366,11 @@ public: DCHECK_NE(ClassId, 0U); bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); Block = TSD->getCache().allocate(ClassId); - // If the allocation failed, the most likely reason with a 32-bit primary - // is the region being full. In that event, retry in each successively - // larger class until it fits. If it fails to fit in the largest class, - // fallback to the Secondary. + // If the allocation failed, retry in each successively larger class until + // it fits. If it fails to fit in the largest class, fallback to the + // Secondary. if (UNLIKELY(!Block)) { while (ClassId < SizeClassMap::LargestClassId && !Block) Block = TSD->getCache().allocate(++ClassId); @@ -406,6 +388,7 @@ public: if (UNLIKELY(!Block)) { if (Options.get(OptionBit::MayReturnNull)) return nullptr; + printStats(); reportOutOfMemory(NeededSize); } @@ -535,14 +518,14 @@ public: Chunk::SizeOrUnusedBytesMask; Chunk::storeHeader(Cookie, Ptr, &Header); - if (UNLIKELY(&__scudo_allocate_hook)) - __scudo_allocate_hook(TaggedPtr, Size); - return TaggedPtr; } NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0, UNUSED uptr Alignment = MinAlignment) { + if (UNLIKELY(!Ptr)) + return; + // For a deallocation, we only ensure minimal initialization, meaning thread // local data will be left uninitialized for now (when using ELF TLS). The // fallback cache will be used instead. This is a workaround for a situation @@ -551,12 +534,6 @@ public: // being destroyed properly. Any other heap operation will do a full init. initThreadMaybe(/*MinimalInit=*/true); - if (UNLIKELY(&__scudo_deallocate_hook)) - __scudo_deallocate_hook(Ptr); - - if (UNLIKELY(!Ptr)) - return; - #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { GuardedAlloc.deallocate(Ptr); @@ -635,47 +612,46 @@ public: if (UNLIKELY(!isAligned(reinterpret_cast(OldPtr), MinAlignment))) reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr); - Chunk::UnpackedHeader OldHeader; - Chunk::loadHeader(Cookie, OldPtr, &OldHeader); + Chunk::UnpackedHeader Header; + Chunk::loadHeader(Cookie, OldPtr, &Header); - if (UNLIKELY(OldHeader.State != Chunk::State::Allocated)) + if (UNLIKELY(Header.State != Chunk::State::Allocated)) reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr); // Pointer has to be allocated with a malloc-type function. Some // applications think that it is OK to realloc a memalign'ed pointer, which // will trigger this check. It really isn't. if (Options.get(OptionBit::DeallocTypeMismatch)) { - if (UNLIKELY(OldHeader.OriginOrWasZeroed != Chunk::Origin::Malloc)) + if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc)) reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr, - OldHeader.OriginOrWasZeroed, + Header.OriginOrWasZeroed, Chunk::Origin::Malloc); } - void *BlockBegin = getBlockBegin(OldTaggedPtr, &OldHeader); + void *BlockBegin = getBlockBegin(OldTaggedPtr, &Header); uptr BlockEnd; uptr OldSize; - const uptr ClassId = OldHeader.ClassId; + const uptr ClassId = Header.ClassId; if (LIKELY(ClassId)) { BlockEnd = reinterpret_cast(BlockBegin) + SizeClassMap::getSizeByClassId(ClassId); - OldSize = OldHeader.SizeOrUnusedBytes; + OldSize = Header.SizeOrUnusedBytes; } else { BlockEnd = SecondaryT::getBlockEnd(BlockBegin); OldSize = BlockEnd - (reinterpret_cast(OldTaggedPtr) + - OldHeader.SizeOrUnusedBytes); + Header.SizeOrUnusedBytes); } // If the new chunk still fits in the previously allocated block (with a // reasonable delta), we just keep the old block, and update the chunk // header to reflect the size change. if (reinterpret_cast(OldTaggedPtr) + NewSize <= BlockEnd) { if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) { - Chunk::UnpackedHeader NewHeader = OldHeader; - NewHeader.SizeOrUnusedBytes = + Header.SizeOrUnusedBytes = (ClassId ? NewSize : BlockEnd - (reinterpret_cast(OldTaggedPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; - Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); + Chunk::storeHeader(Cookie, OldPtr, &Header); if (UNLIKELY(useMemoryTagging(Options))) { if (ClassId) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, @@ -697,9 +673,7 @@ public: void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment); if (LIKELY(NewPtr)) { memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize)); - if (UNLIKELY(&__scudo_deallocate_hook)) - __scudo_deallocate_hook(OldTaggedPtr); - quarantineOrDeallocateChunk(Options, OldTaggedPtr, &OldHeader, OldSize); + quarantineOrDeallocateChunk(Options, OldTaggedPtr, &Header, OldSize); } return NewPtr; } @@ -754,6 +728,13 @@ public: Str.output(); } + void printFragmentationInfo() { + ScopedString Str; + Primary.getFragmentationInfo(&Str); + // Secondary allocator dumps the fragmentation data in getStats(). + Str.output(); + } + void releaseToOS(ReleaseToOS ReleaseType) { initThreadMaybe(); if (ReleaseType == ReleaseToOS::ForceAll) @@ -847,10 +828,15 @@ public: // for it, which then forces realloc to copy the usable size of a chunk as // opposed to its actual size. uptr getUsableSize(const void *Ptr) { - initThreadMaybe(); if (UNLIKELY(!Ptr)) return 0; + return getAllocSize(Ptr); + } + + uptr getAllocSize(const void *Ptr) { + initThreadMaybe(); + #ifdef GWP_ASAN_HOOKS if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) return GuardedAlloc.getSize(Ptr); @@ -859,9 +845,11 @@ public: Ptr = getHeaderTaggedPointer(const_cast(Ptr)); Chunk::UnpackedHeader Header; Chunk::loadHeader(Cookie, Ptr, &Header); - // Getting the usable size of a chunk only makes sense if it's allocated. + + // Getting the alloc size of a chunk only makes sense if it's allocated. if (UNLIKELY(Header.State != Chunk::State::Allocated)) reportInvalidChunkState(AllocatorAction::Sizing, const_cast(Ptr)); + return getSize(Ptr, &Header); } @@ -887,13 +875,6 @@ public: Header.State == Chunk::State::Allocated; } - void setRssLimitsTestOnly(int SoftRssLimitMb, int HardRssLimitMb, - bool MayReturnNull) { - RssChecker.init(SoftRssLimitMb, HardRssLimitMb); - if (MayReturnNull) - Primary.Options.set(OptionBit::MayReturnNull); - } - bool useMemoryTaggingTestOnly() const { return useMemoryTagging(Primary.Options.load()); } @@ -913,7 +894,7 @@ public: void setTrackAllocationStacks(bool Track) { initThreadMaybe(); - if (getFlags()->allocation_ring_buffer_size == 0) { + if (getFlags()->allocation_ring_buffer_size <= 0) { DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks)); return; } @@ -955,21 +936,7 @@ public: uptr getRingBufferSize() { initThreadMaybe(); - auto *RingBuffer = getRingBuffer(); - return RingBuffer ? ringBufferSizeInBytes(RingBuffer->Size) : 0; - } - - static bool setRingBufferSizeForBuffer(char *Buffer, size_t Size) { - // Need at least one entry. - if (Size < sizeof(AllocationRingBuffer) + - sizeof(typename AllocationRingBuffer::Entry)) { - return false; - } - AllocationRingBuffer *RingBuffer = - reinterpret_cast(Buffer); - RingBuffer->Size = (Size - sizeof(AllocationRingBuffer)) / - sizeof(typename AllocationRingBuffer::Entry); - return true; + return RingBufferElements ? ringBufferSizeInBytes(RingBufferElements) : 0; } static const uptr MaxTraceSize = 64; @@ -986,8 +953,9 @@ public: static void getErrorInfo(struct scudo_error_info *ErrorInfo, uintptr_t FaultAddr, const char *DepotPtr, const char *RegionInfoPtr, const char *RingBufferPtr, - const char *Memory, const char *MemoryTags, - uintptr_t MemoryAddr, size_t MemorySize) { + size_t RingBufferSize, const char *Memory, + const char *MemoryTags, uintptr_t MemoryAddr, + size_t MemorySize) { *ErrorInfo = {}; if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) @@ -1006,7 +974,7 @@ public: // Check the ring buffer. For primary allocations this will only find UAF; // for secondary allocations we can find either UAF or OOB. getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot, - RingBufferPtr); + RingBufferPtr, RingBufferSize); // Check for OOB in the 28 blocks surrounding the 3 we checked earlier. // Beyond that we are likely to hit false positives. @@ -1053,7 +1021,6 @@ private: QuarantineT Quarantine; TSDRegistryT TSDRegistry; pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT; - RssLimitChecker RssChecker; #ifdef GWP_ASAN_HOOKS gwp_asan::GuardedPoolAllocator GuardedAlloc; @@ -1073,13 +1040,14 @@ private: }; atomic_uptr Pos; - u32 Size; // An array of Size (at least one) elements of type Entry is immediately // following to this struct. }; // Pointer to memory mapped area starting with AllocationRingBuffer struct, // and immediately followed by Size elements of type Entry. char *RawRingBuffer = {}; + u32 RingBufferElements = 0; + MemMapT RawRingBufferMap; // The following might get optimized out by the compiler. NOINLINE void performSanityChecks() { @@ -1134,35 +1102,34 @@ private: reinterpret_cast(Ptr) - SizeOrUnusedBytes; } - void quarantineOrDeallocateChunk(Options Options, void *TaggedPtr, + void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr, Chunk::UnpackedHeader *Header, uptr Size) NO_THREAD_SAFETY_ANALYSIS { void *Ptr = getHeaderTaggedPointer(TaggedPtr); - Chunk::UnpackedHeader NewHeader = *Header; // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. // This purposefully underflows for Size == 0. const bool BypassQuarantine = !Quarantine.getCacheSize() || ((Size - 1) >= QuarantineMaxChunkSize) || - !NewHeader.ClassId; + !Header->ClassId; if (BypassQuarantine) - NewHeader.State = Chunk::State::Available; + Header->State = Chunk::State::Available; else - NewHeader.State = Chunk::State::Quarantined; - NewHeader.OriginOrWasZeroed = useMemoryTagging(Options) && - NewHeader.ClassId && - !TSDRegistry.getDisableMemInit(); - Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); + Header->State = Chunk::State::Quarantined; + Header->OriginOrWasZeroed = useMemoryTagging(Options) && + Header->ClassId && + !TSDRegistry.getDisableMemInit(); + Chunk::storeHeader(Cookie, Ptr, Header); if (UNLIKELY(useMemoryTagging(Options))) { u8 PrevTag = extractTag(reinterpret_cast(TaggedPtr)); storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size); - if (NewHeader.ClassId) { + if (Header->ClassId) { if (!TSDRegistry.getDisableMemInit()) { uptr TaggedBegin, TaggedEnd; const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe( - Options, reinterpret_cast(getBlockBegin(Ptr, &NewHeader)), - NewHeader.ClassId); + Options, reinterpret_cast(getBlockBegin(Ptr, Header)), + Header->ClassId); // Exclude the previous tag so that immediate use after free is // detected 100% of the time. setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin, @@ -1173,11 +1140,12 @@ private: if (BypassQuarantine) { if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); - void *BlockBegin = getBlockBegin(Ptr, &NewHeader); - const uptr ClassId = NewHeader.ClassId; + void *BlockBegin = getBlockBegin(Ptr, Header); + const uptr ClassId = Header->ClassId; if (LIKELY(ClassId)) { bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); const bool CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin); if (UnlockRequired) @@ -1197,6 +1165,7 @@ private: } else { bool UnlockRequired; auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); Quarantine.put(&TSD->getQuarantineCache(), QuarantineCallback(*this, TSD->getCache()), Ptr, Size); if (UnlockRequired) @@ -1273,7 +1242,7 @@ private: storeEndMarker(RoundNewPtr, NewSize, BlockEnd); } - void storePrimaryAllocationStackMaybe(Options Options, void *Ptr) { + void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; auto *Ptr32 = reinterpret_cast(Ptr); @@ -1286,7 +1255,7 @@ private: u32 DeallocationTid) { uptr Pos = atomic_fetch_add(&getRingBuffer()->Pos, 1, memory_order_relaxed); typename AllocationRingBuffer::Entry *Entry = - getRingBufferEntry(RawRingBuffer, Pos % getRingBuffer()->Size); + getRingBufferEntry(RawRingBuffer, Pos % RingBufferElements); // First invalidate our entry so that we don't attempt to interpret a // partially written state in getSecondaryErrorInfo(). The fences below @@ -1305,7 +1274,7 @@ private: atomic_store_relaxed(&Entry->Ptr, reinterpret_cast(Ptr)); } - void storeSecondaryAllocationStackMaybe(Options Options, void *Ptr, + void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr, uptr Size) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; @@ -1320,8 +1289,8 @@ private: storeRingBufferEntry(untagPointer(Ptr), Trace, Tid, Size, 0, 0); } - void storeDeallocationStackMaybe(Options Options, void *Ptr, u8 PrevTag, - uptr Size) { + void storeDeallocationStackMaybe(const Options &Options, void *Ptr, + u8 PrevTag, uptr Size) { if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks))) return; @@ -1427,17 +1396,19 @@ private: size_t &NextErrorReport, uintptr_t FaultAddr, const StackDepot *Depot, - const char *RingBufferPtr) { + const char *RingBufferPtr, + size_t RingBufferSize) { auto *RingBuffer = reinterpret_cast(RingBufferPtr); - if (!RingBuffer || RingBuffer->Size == 0) + size_t RingBufferElements = ringBufferElementsFromBytes(RingBufferSize); + if (!RingBuffer || RingBufferElements == 0) return; uptr Pos = atomic_load_relaxed(&RingBuffer->Pos); - for (uptr I = Pos - 1; - I != Pos - 1 - RingBuffer->Size && NextErrorReport != NumErrorReports; + for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements && + NextErrorReport != NumErrorReports; --I) { - auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBuffer->Size); + auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBufferElements); uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr); if (!EntryPtr) continue; @@ -1516,17 +1487,19 @@ private: } void mapAndInitializeRingBuffer() { + if (getFlags()->allocation_ring_buffer_size <= 0) + return; u32 AllocationRingBufferSize = static_cast(getFlags()->allocation_ring_buffer_size); - if (AllocationRingBufferSize < 1) - return; - RawRingBuffer = static_cast( - map(/*Addr=*/nullptr, - roundUp(ringBufferSizeInBytes(AllocationRingBufferSize), - getPageSizeCached()), - "AllocatorRingBuffer")); - auto *RingBuffer = reinterpret_cast(RawRingBuffer); - RingBuffer->Size = AllocationRingBufferSize; + MemMapT MemMap; + MemMap.map( + /*Addr=*/0U, + roundUp(ringBufferSizeInBytes(AllocationRingBufferSize), + getPageSizeCached()), + "scudo:ring_buffer"); + RawRingBuffer = reinterpret_cast(MemMap.getBase()); + RawRingBufferMap = MemMap; + RingBufferElements = AllocationRingBufferSize; static_assert(sizeof(AllocationRingBuffer) % alignof(typename AllocationRingBuffer::Entry) == 0, @@ -1534,14 +1507,25 @@ private: } void unmapRingBuffer() { - unmap(RawRingBuffer, roundUp(getRingBufferSize(), getPageSizeCached())); + auto *RingBuffer = getRingBuffer(); + if (RingBuffer != nullptr) { + RawRingBufferMap.unmap(RawRingBufferMap.getBase(), + RawRingBufferMap.getCapacity()); + } RawRingBuffer = nullptr; } - static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) { + static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) { return sizeof(AllocationRingBuffer) + - AllocationRingBufferSize * - sizeof(typename AllocationRingBuffer::Entry); + RingBufferElements * sizeof(typename AllocationRingBuffer::Entry); + } + + static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) { + if (Bytes < sizeof(AllocationRingBuffer)) { + return 0; + } + return (Bytes - sizeof(AllocationRingBuffer)) / + sizeof(typename AllocationRingBuffer::Entry); } inline AllocationRingBuffer *getRingBuffer() { diff --git a/Telegram/ThirdParty/scudo/common.cpp b/Telegram/ThirdParty/scudo/common.cpp index 9f14faeef..06e930638 100644 --- a/Telegram/ThirdParty/scudo/common.cpp +++ b/Telegram/ThirdParty/scudo/common.cpp @@ -21,22 +21,4 @@ uptr getPageSizeSlow() { return PageSizeCached; } -// Fatal internal map() or unmap() error (potentially OOM related). -void NORETURN dieOnMapUnmapError(uptr SizeIfOOM) { - char Error[128] = "Scudo ERROR: internal map or unmap failure\n"; - if (SizeIfOOM) { - formatString( - Error, sizeof(Error), - "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n", - SizeIfOOM >> 10); - } - outputRaw(Error); - setAbortMessage(Error); - die(); -} - -#if !SCUDO_LINUX -uptr GetRSS() { return 0; } -#endif - } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/common.h b/Telegram/ThirdParty/scudo/common.h index 82e6cf4ae..ae45683f1 100644 --- a/Telegram/ThirdParty/scudo/common.h +++ b/Telegram/ThirdParty/scudo/common.h @@ -17,6 +17,7 @@ #include #include +#include namespace scudo { @@ -111,19 +112,19 @@ template inline void shuffle(T *A, u32 N, u32 *RandState) { *RandState = State; } -// Hardware specific inlinable functions. +inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral, + uptr *Fractional) { + constexpr uptr Digits = 100; + if (Denominator == 0) { + *Integral = 100; + *Fractional = 0; + return; + } -inline void yieldProcessor(UNUSED u8 Count) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("" ::: "memory"); - for (u8 I = 0; I < Count; I++) - __asm__ __volatile__("pause"); -#elif defined(__aarch64__) || defined(__arm__) - __asm__ __volatile__("" ::: "memory"); - for (u8 I = 0; I < Count; I++) - __asm__ __volatile__("yield"); -#endif - __asm__ __volatile__("" ::: "memory"); + *Integral = Numerator * Digits / Denominator; + *Fractional = + (((Numerator * Digits) % Denominator) * Digits + Denominator / 2) / + Denominator; } // Platform specific functions. @@ -131,9 +132,10 @@ inline void yieldProcessor(UNUSED u8 Count) { extern uptr PageSizeCached; uptr getPageSizeSlow(); inline uptr getPageSizeCached() { - // Bionic uses a hardcoded value. - if (SCUDO_ANDROID) - return 4096U; +#if SCUDO_ANDROID && defined(PAGE_SIZE) + // Most Android builds have a build-time constant page size. + return PAGE_SIZE; +#endif if (LIKELY(PageSizeCached)) return PageSizeCached; return getPageSizeSlow(); @@ -144,8 +146,6 @@ u32 getNumberOfCPUs(); const char *getEnv(const char *Name); -uptr GetRSS(); - u64 getMonotonicTime(); // Gets the time faster but with less accuracy. Can call getMonotonicTime // if no fast version is available. @@ -190,10 +190,6 @@ void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, MapPlatformData *Data = nullptr); -// Internal map & unmap fatal error. This must not call map(). SizeIfOOM shall -// hold the requested size on an out-of-memory error, 0 otherwise. -void NORETURN dieOnMapUnmapError(uptr SizeIfOOM = 0); - // Logging related functions. void setAbortMessage(const char *Message); diff --git a/Telegram/ThirdParty/scudo/condition_variable.h b/Telegram/ThirdParty/scudo/condition_variable.h new file mode 100644 index 000000000..4afebdc9d --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable.h @@ -0,0 +1,60 @@ +//===-- condition_variable.h ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_H_ +#define SCUDO_CONDITION_VARIABLE_H_ + +#include "condition_variable_base.h" + +#include "common.h" +#include "platform.h" + +#include "condition_variable_linux.h" + +namespace scudo { + +// A default implementation of default condition variable. It doesn't do a real +// `wait`, instead it spins a short amount of time only. +class ConditionVariableDummy + : public ConditionVariableBase { +public: + void notifyAllImpl(UNUSED HybridMutex &M) REQUIRES(M) {} + + void waitImpl(UNUSED HybridMutex &M) REQUIRES(M) { + M.unlock(); + + constexpr u32 SpinTimes = 64; + volatile u32 V = 0; + for (u32 I = 0; I < SpinTimes; ++I) { + u32 Tmp = V + 1; + V = Tmp; + } + + M.lock(); + } +}; + +template +struct ConditionVariableState { + static constexpr bool enabled() { return false; } + // This is only used for compilation purpose so that we won't end up having + // many conditional compilations. If you want to use `ConditionVariableDummy`, + // define `ConditionVariableT` in your allocator configuration. See + // allocator_config.h for more details. + using ConditionVariableT = ConditionVariableDummy; +}; + +template +struct ConditionVariableState { + static constexpr bool enabled() { return Config::UseConditionVariable; } + using ConditionVariableT = typename Config::ConditionVariableT; +}; + +} // namespace scudo + +#endif // SCUDO_CONDITION_VARIABLE_H_ diff --git a/Telegram/ThirdParty/scudo/condition_variable_base.h b/Telegram/ThirdParty/scudo/condition_variable_base.h new file mode 100644 index 000000000..416c327fe --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_base.h @@ -0,0 +1,56 @@ +//===-- condition_variable_base.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_BASE_H_ +#define SCUDO_CONDITION_VARIABLE_BASE_H_ + +#include "mutex.h" +#include "thread_annotations.h" + +namespace scudo { + +template class ConditionVariableBase { +public: + constexpr ConditionVariableBase() = default; + + void bindTestOnly(HybridMutex &Mutex) { +#if SCUDO_DEBUG + boundMutex = &Mutex; +#else + (void)Mutex; +#endif + } + + void notifyAll(HybridMutex &M) REQUIRES(M) { +#if SCUDO_DEBUG + CHECK_EQ(&M, boundMutex); +#endif + getDerived()->notifyAllImpl(M); + } + + void wait(HybridMutex &M) REQUIRES(M) { +#if SCUDO_DEBUG + CHECK_EQ(&M, boundMutex); +#endif + getDerived()->waitImpl(M); + } + +protected: + Derived *getDerived() { return static_cast(this); } + +#if SCUDO_DEBUG + // Because thread-safety analysis doesn't support pointer aliasing, we are not + // able to mark the proper annotations without false positive. Instead, we + // pass the lock and do the same-lock check separately. + HybridMutex *boundMutex = nullptr; +#endif +}; + +} // namespace scudo + +#endif // SCUDO_CONDITION_VARIABLE_BASE_H_ diff --git a/Telegram/ThirdParty/scudo/condition_variable_linux.cpp b/Telegram/ThirdParty/scudo/condition_variable_linux.cpp new file mode 100644 index 000000000..e6d9bd177 --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_linux.cpp @@ -0,0 +1,52 @@ +//===-- condition_variable_linux.cpp ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "condition_variable_linux.h" + +#include "atomic_helpers.h" + +#include +#include +#include +#include + +namespace scudo { + +void ConditionVariableLinux::notifyAllImpl(UNUSED HybridMutex &M) { + const u32 V = atomic_load_relaxed(&Counter); + atomic_store_relaxed(&Counter, V + 1); + + // TODO(chiahungduan): Move the waiters from the futex waiting queue + // `Counter` to futex waiting queue `M` so that the awoken threads won't be + // blocked again due to locked `M` by current thread. + if (LastNotifyAll != V) { + syscall(SYS_futex, reinterpret_cast(&Counter), FUTEX_WAKE_PRIVATE, + INT_MAX, nullptr, nullptr, 0); + } + + LastNotifyAll = V + 1; +} + +void ConditionVariableLinux::waitImpl(HybridMutex &M) { + const u32 V = atomic_load_relaxed(&Counter) + 1; + atomic_store_relaxed(&Counter, V); + + // TODO: Use ScopedUnlock when it's supported. + M.unlock(); + syscall(SYS_futex, reinterpret_cast(&Counter), FUTEX_WAIT_PRIVATE, V, + nullptr, nullptr, 0); + M.lock(); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/Telegram/ThirdParty/scudo/condition_variable_linux.h b/Telegram/ThirdParty/scudo/condition_variable_linux.h new file mode 100644 index 000000000..cd0732873 --- /dev/null +++ b/Telegram/ThirdParty/scudo/condition_variable_linux.h @@ -0,0 +1,38 @@ +//===-- condition_variable_linux.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CONDITION_VARIABLE_LINUX_H_ +#define SCUDO_CONDITION_VARIABLE_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +#include "atomic_helpers.h" +#include "condition_variable_base.h" +#include "thread_annotations.h" + +namespace scudo { + +class ConditionVariableLinux + : public ConditionVariableBase { +public: + void notifyAllImpl(HybridMutex &M) REQUIRES(M); + + void waitImpl(HybridMutex &M) REQUIRES(M); + +private: + u32 LastNotifyAll = 0; + atomic_u32 Counter = {}; +}; + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_CONDITION_VARIABLE_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/flags.cpp b/Telegram/ThirdParty/scudo/flags.cpp index de5153b28..f498edfbd 100644 --- a/Telegram/ThirdParty/scudo/flags.cpp +++ b/Telegram/ThirdParty/scudo/flags.cpp @@ -68,6 +68,9 @@ void initFlags() { Parser.parseString(getCompileDefinitionScudoDefaultOptions()); Parser.parseString(getScudoDefaultOptions()); Parser.parseString(getEnv("SCUDO_OPTIONS")); + if (const char *V = getEnv("SCUDO_ALLOCATION_RING_BUFFER_SIZE")) { + Parser.parseStringPair("allocation_ring_buffer_size", V); + } } } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/flags.inc b/Telegram/ThirdParty/scudo/flags.inc index c1f153baf..f5a2bab50 100644 --- a/Telegram/ThirdParty/scudo/flags.inc +++ b/Telegram/ThirdParty/scudo/flags.inc @@ -46,14 +46,6 @@ SCUDO_FLAG(int, release_to_os_interval_ms, SCUDO_ANDROID ? INT32_MIN : 5000, "Interval (in milliseconds) at which to attempt release of unused " "memory to the OS. Negative values disable the feature.") -SCUDO_FLAG(int, hard_rss_limit_mb, 0, - "Hard RSS Limit in Mb. If non-zero, once the limit is achieved, " - "abort the process") - -SCUDO_FLAG(int, soft_rss_limit_mb, 0, - "Soft RSS Limit in Mb. If non-zero, once the limit is reached, all " - "subsequent calls will fail or return NULL until the RSS goes below " - "the soft limit") - SCUDO_FLAG(int, allocation_ring_buffer_size, 32768, - "Entries to keep in the allocation ring buffer for scudo.") + "Entries to keep in the allocation ring buffer for scudo. " + "Values less or equal to zero disable the buffer.") diff --git a/Telegram/ThirdParty/scudo/flags_parser.cpp b/Telegram/ThirdParty/scudo/flags_parser.cpp index be39fcd4f..3d8c6f378 100644 --- a/Telegram/ThirdParty/scudo/flags_parser.cpp +++ b/Telegram/ThirdParty/scudo/flags_parser.cpp @@ -10,6 +10,8 @@ #include "common.h" #include "report.h" +#include +#include #include #include @@ -80,7 +82,7 @@ void FlagParser::parseFlag() { ++Pos; Value = Buffer + ValueStart; } - if (!runHandler(Name, Value)) + if (!runHandler(Name, Value, '=')) reportError("flag parsing failed."); } @@ -122,10 +124,16 @@ inline bool parseBool(const char *Value, bool *b) { return false; } -bool FlagParser::runHandler(const char *Name, const char *Value) { +void FlagParser::parseStringPair(const char *Name, const char *Value) { + if (!runHandler(Name, Value, '\0')) + reportError("flag parsing failed."); +} + +bool FlagParser::runHandler(const char *Name, const char *Value, + const char Sep) { for (u32 I = 0; I < NumberOfFlags; ++I) { const uptr Len = strlen(Flags[I].Name); - if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != '=') + if (strncmp(Name, Flags[I].Name, Len) != 0 || Name[Len] != Sep) continue; bool Ok = false; switch (Flags[I].Type) { @@ -136,12 +144,18 @@ bool FlagParser::runHandler(const char *Name, const char *Value) { break; case FlagType::FT_int: char *ValueEnd; - *reinterpret_cast(Flags[I].Var) = - static_cast(strtol(Value, &ValueEnd, 10)); - Ok = - *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd); - if (!Ok) + errno = 0; + long V = strtol(Value, &ValueEnd, 10); + if (errno != 0 || // strtol failed (over or underflow) + V > INT_MAX || V < INT_MIN || // overflows integer + // contains unexpected characters + (*ValueEnd != '"' && *ValueEnd != '\'' && + !isSeparatorOrNull(*ValueEnd))) { reportInvalidFlag("int", Value); + break; + } + *reinterpret_cast(Flags[I].Var) = static_cast(V); + Ok = true; break; } return Ok; diff --git a/Telegram/ThirdParty/scudo/flags_parser.h b/Telegram/ThirdParty/scudo/flags_parser.h index ba832adbd..ded496fda 100644 --- a/Telegram/ThirdParty/scudo/flags_parser.h +++ b/Telegram/ThirdParty/scudo/flags_parser.h @@ -27,6 +27,7 @@ public: void *Var); void parseString(const char *S); void printFlagDescriptions(); + void parseStringPair(const char *Name, const char *Value); private: static const u32 MaxFlags = 20; @@ -45,7 +46,7 @@ private: void skipWhitespace(); void parseFlags(); void parseFlag(); - bool runHandler(const char *Name, const char *Value); + bool runHandler(const char *Name, const char *Value, char Sep); }; void reportUnrecognizedFlags(); diff --git a/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp b/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp index 74456450a..5b01ebe11 100644 --- a/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp +++ b/Telegram/ThirdParty/scudo/fuzz/get_error_info_fuzzer.cpp @@ -46,14 +46,11 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) { } std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString(); - // RingBuffer is too short. - if (!AllocatorT::setRingBufferSizeForBuffer(RingBufferBytes.data(), - RingBufferBytes.size())) - return 0; scudo_error_info ErrorInfo; AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(), - RegionInfo.data(), RingBufferBytes.data(), Memory, - MemoryTags, MemoryAddr, MemorySize); + RegionInfo.data(), RingBufferBytes.data(), + RingBufferBytes.size(), Memory, MemoryTags, + MemoryAddr, MemorySize); return 0; } diff --git a/Telegram/ThirdParty/scudo/include/scudo/interface.h b/Telegram/ThirdParty/scudo/include/scudo/interface.h index 6c0c521f8..a2dedea91 100644 --- a/Telegram/ThirdParty/scudo/include/scudo/interface.h +++ b/Telegram/ThirdParty/scudo/include/scudo/interface.h @@ -17,10 +17,22 @@ extern "C" { __attribute__((weak)) const char *__scudo_default_options(void); // Post-allocation & pre-deallocation hooks. -// They must be thread-safe and not use heap related functions. __attribute__((weak)) void __scudo_allocate_hook(void *ptr, size_t size); __attribute__((weak)) void __scudo_deallocate_hook(void *ptr); +// `realloc` involves both deallocation and allocation but they are not reported +// atomically. In one specific case which may keep taking a snapshot right in +// the middle of `realloc` reporting the deallocation and allocation, it may +// confuse the user by missing memory from `realloc`. To alleviate that case, +// define the two `realloc` hooks to get the knowledge of the bundled hook +// calls. These hooks are optional and should only be used when a hooks user +// wants to track reallocs more closely. +// +// See more details in the comment of `realloc` in wrapper_c.inc. +__attribute__((weak)) void +__scudo_realloc_allocate_hook(void *old_ptr, void *new_ptr, size_t size); +__attribute__((weak)) void __scudo_realloc_deallocate_hook(void *old_ptr); + void __scudo_print_stats(void); typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); @@ -73,7 +85,8 @@ typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); // pointer. void __scudo_get_error_info(struct scudo_error_info *error_info, uintptr_t fault_addr, const char *stack_depot, - const char *region_info, const char *ring_buffer, + size_t stack_depot_size, const char *region_info, + const char *ring_buffer, size_t ring_buffer_size, const char *memory, const char *memory_tags, uintptr_t memory_addr, size_t memory_size); diff --git a/Telegram/ThirdParty/scudo/linux.cpp b/Telegram/ThirdParty/scudo/linux.cpp index e285d8a3d..274695108 100644 --- a/Telegram/ThirdParty/scudo/linux.cpp +++ b/Telegram/ThirdParty/scudo/linux.cpp @@ -14,6 +14,7 @@ #include "internal_defs.h" #include "linux.h" #include "mutex.h" +#include "report_linux.h" #include "string_utils.h" #include @@ -43,6 +44,7 @@ uptr getPageSize() { return static_cast(sysconf(_SC_PAGESIZE)); } void NORETURN die() { abort(); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, UNUSED MapPlatformData *Data) { int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS; @@ -65,7 +67,7 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0); if (P == MAP_FAILED) { if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) - dieOnMapUnmapError(errno == ENOMEM ? Size : 0); + reportMapError(errno == ENOMEM ? Size : 0); return nullptr; } #if SCUDO_ANDROID @@ -75,19 +77,22 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, return P; } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void unmap(void *Addr, uptr Size, UNUSED uptr Flags, UNUSED MapPlatformData *Data) { if (munmap(Addr, Size) != 0) - dieOnMapUnmapError(); + reportUnmapError(reinterpret_cast(Addr), Size); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, UNUSED MapPlatformData *Data) { int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) - dieOnMapUnmapError(); + reportProtectError(Addr, Size, Prot); } +// TODO: Will be deprecated. Use the interfaces in MemMapLinux instead. void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, UNUSED MapPlatformData *Data) { void *Addr = reinterpret_cast(BaseAddress + Offset); @@ -104,12 +109,14 @@ enum State : u32 { Unlocked = 0, Locked = 1, Sleeping = 2 }; } bool HybridMutex::tryLock() { - return atomic_compare_exchange(&M, Unlocked, Locked) == Unlocked; + return atomic_compare_exchange_strong(&M, Unlocked, Locked, + memory_order_acquire) == Unlocked; } // The following is based on https://akkadia.org/drepper/futex.pdf. void HybridMutex::lockSlow() { - u32 V = atomic_compare_exchange(&M, Unlocked, Locked); + u32 V = atomic_compare_exchange_strong(&M, Unlocked, Locked, + memory_order_acquire); if (V == Unlocked) return; if (V != Sleeping) @@ -197,39 +204,6 @@ bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) { extern "C" WEAK int async_safe_write_log(int pri, const char *tag, const char *msg); -static uptr GetRSSFromBuffer(const char *Buf) { - // The format of the file is: - // 1084 89 69 11 0 79 0 - // We need the second number which is RSS in pages. - const char *Pos = Buf; - // Skip the first number. - while (*Pos >= '0' && *Pos <= '9') - Pos++; - // Skip whitespaces. - while (!(*Pos >= '0' && *Pos <= '9') && *Pos != 0) - Pos++; - // Read the number. - u64 Rss = 0; - for (; *Pos >= '0' && *Pos <= '9'; Pos++) - Rss = Rss * 10 + static_cast(*Pos) - '0'; - return static_cast(Rss * getPageSizeCached()); -} - -uptr GetRSS() { - // TODO: We currently use sanitizer_common's GetRSS which reads the - // RSS from /proc/self/statm by default. We might want to - // call getrusage directly, even if it's less accurate. - auto Fd = open("/proc/self/statm", O_RDONLY); - char Buf[64]; - s64 Len = read(Fd, Buf, sizeof(Buf) - 1); - close(Fd); - if (Len <= 0) - return 0; - Buf[Len] = 0; - - return GetRSSFromBuffer(Buf); -} - void outputRaw(const char *Buffer) { if (&async_safe_write_log) { constexpr s32 AndroidLogInfo = 4; diff --git a/Telegram/ThirdParty/scudo/local_cache.h b/Telegram/ThirdParty/scudo/local_cache.h index 1095eb5f1..46d6affdc 100644 --- a/Telegram/ThirdParty/scudo/local_cache.h +++ b/Telegram/ThirdParty/scudo/local_cache.h @@ -22,80 +22,13 @@ template struct SizeClassAllocatorLocalCache { typedef typename SizeClassAllocator::SizeClassMap SizeClassMap; typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; - struct TransferBatch { - static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint; - void setFromArray(CompactPtrT *Array, u16 N) { - DCHECK_LE(N, MaxNumCached); - Count = N; - memcpy(Batch, Array, sizeof(Batch[0]) * Count); - } - void appendFromArray(CompactPtrT *Array, u16 N) { - DCHECK_LE(N, MaxNumCached - Count); - memcpy(Batch + Count, Array, sizeof(Batch[0]) * N); - // u16 will be promoted to int by arithmetic type conversion. - Count = static_cast(Count + N); - } - void appendFromTransferBatch(TransferBatch *B, u16 N) { - DCHECK_LE(N, MaxNumCached - Count); - DCHECK_GE(B->Count, N); - // Append from the back of `B`. - memcpy(Batch + Count, B->Batch + (B->Count - N), sizeof(Batch[0]) * N); - // u16 will be promoted to int by arithmetic type conversion. - Count = static_cast(Count + N); - B->Count = static_cast(B->Count - N); - } - void clear() { Count = 0; } - void add(CompactPtrT P) { - DCHECK_LT(Count, MaxNumCached); - Batch[Count++] = P; - } - void copyToArray(CompactPtrT *Array) const { - memcpy(Array, Batch, sizeof(Batch[0]) * Count); - } - u16 getCount() const { return Count; } - bool isEmpty() const { return Count == 0U; } - CompactPtrT get(u16 I) const { - DCHECK_LE(I, Count); - return Batch[I]; - } - static u16 getMaxCached(uptr Size) { - return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size)); - } - TransferBatch *Next; - - private: - CompactPtrT Batch[MaxNumCached]; - u16 Count; - }; - - // A BatchGroup is used to collect blocks. Each group has a group id to - // identify the group kind of contained blocks. - struct BatchGroup { - // `Next` is used by IntrusiveList. - BatchGroup *Next; - // The compact base address of each group - uptr CompactPtrGroupBase; - // Cache value of TransferBatch::getMaxCached() - u16 MaxCachedPerBatch; - // Number of blocks pushed into this group. This is an increment-only - // counter. - uptr PushedBlocks; - // This is used to track how many bytes are not in-use since last time we - // tried to release pages. - uptr BytesInBGAtLastCheckpoint; - // Blocks are managed by TransferBatch in a list. - SinglyLinkedList Batches; - }; - - static_assert(sizeof(BatchGroup) <= sizeof(TransferBatch), - "BatchGroup uses the same class size as TransferBatch"); - void init(GlobalStats *S, SizeClassAllocator *A) { DCHECK(isEmpty()); Stats.init(); if (LIKELY(S)) S->link(&Stats); Allocator = A; + initCache(); } void destroy(GlobalStats *S) { @@ -108,7 +41,9 @@ template struct SizeClassAllocatorLocalCache { DCHECK_LT(ClassId, NumClasses); PerClass *C = &PerClassArray[ClassId]; if (C->Count == 0) { - if (UNLIKELY(!refill(C, ClassId))) + // Refill half of the number of max cached. + DCHECK_GT(C->MaxCount / 2, 0U); + if (UNLIKELY(!refill(C, ClassId, C->MaxCount / 2))) return nullptr; DCHECK_GT(C->Count, 0); } @@ -125,9 +60,6 @@ template struct SizeClassAllocatorLocalCache { bool deallocate(uptr ClassId, void *P) { CHECK_LT(ClassId, NumClasses); PerClass *C = &PerClassArray[ClassId]; - // We still have to initialize the cache in the event that the first heap - // operation in a thread is a deallocation. - initCacheMaybe(C); // If the cache is full, drain half of blocks back to the main allocator. const bool NeedToDrainCache = C->Count == C->MaxCount; @@ -151,7 +83,7 @@ template struct SizeClassAllocatorLocalCache { } void drain() { - // Drain BatchClassId last as createBatch can refill it. + // Drain BatchClassId last as it may be needed while draining normal blocks. for (uptr I = 0; I < NumClasses; ++I) { if (I == BatchClassId) continue; @@ -163,19 +95,11 @@ template struct SizeClassAllocatorLocalCache { DCHECK(isEmpty()); } - TransferBatch *createBatch(uptr ClassId, void *B) { - if (ClassId != BatchClassId) - B = allocate(BatchClassId); + void *getBatchClassBlock() { + void *B = allocate(BatchClassId); if (UNLIKELY(!B)) reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId)); - return reinterpret_cast(B); - } - - BatchGroup *createGroup() { - void *Ptr = allocate(BatchClassId); - if (UNLIKELY(!Ptr)) - reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId)); - return reinterpret_cast(Ptr); + return B; } LocalStats &getStats() { return Stats; } @@ -203,6 +127,11 @@ template struct SizeClassAllocatorLocalCache { Str->append(" No block is cached.\n"); } + static u16 getMaxCached(uptr Size) { + return Min(SizeClassMap::MaxNumCachedHint, + SizeClassMap::getMaxCachedHint(Size)); + } + private: static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr BatchClassId = SizeClassMap::BatchClassId; @@ -211,24 +140,17 @@ private: u16 MaxCount; // Note: ClassSize is zero for the transfer batch. uptr ClassSize; - CompactPtrT Chunks[2 * TransferBatch::MaxNumCached]; + CompactPtrT Chunks[2 * SizeClassMap::MaxNumCachedHint]; }; PerClass PerClassArray[NumClasses] = {}; LocalStats Stats; SizeClassAllocator *Allocator = nullptr; - ALWAYS_INLINE void initCacheMaybe(PerClass *C) { - if (LIKELY(C->MaxCount)) - return; - initCache(); - DCHECK_NE(C->MaxCount, 0U); - } - NOINLINE void initCache() { for (uptr I = 0; I < NumClasses; I++) { PerClass *P = &PerClassArray[I]; const uptr Size = SizeClassAllocator::getSizeByClassId(I); - P->MaxCount = static_cast(2 * TransferBatch::getMaxCached(Size)); + P->MaxCount = static_cast(2 * getMaxCached(Size)); if (I != BatchClassId) { P->ClassSize = Size; } else { @@ -244,17 +166,12 @@ private: deallocate(BatchClassId, B); } - NOINLINE bool refill(PerClass *C, uptr ClassId) { - initCacheMaybe(C); - TransferBatch *B = Allocator->popBatch(this, ClassId); - if (UNLIKELY(!B)) - return false; - DCHECK_GT(B->getCount(), 0); - C->Count = B->getCount(); - B->copyToArray(C->Chunks); - B->clear(); - destroyBatch(ClassId, B); - return true; + NOINLINE bool refill(PerClass *C, uptr ClassId, u16 MaxRefill) { + const u16 NumBlocksRefilled = + Allocator->popBlocks(this, ClassId, C->Chunks, MaxRefill); + DCHECK_LE(NumBlocksRefilled, MaxRefill); + C->Count = static_cast(C->Count + NumBlocksRefilled); + return NumBlocksRefilled != 0; } NOINLINE void drain(PerClass *C, uptr ClassId) { diff --git a/Telegram/ThirdParty/scudo/mem_map.h b/Telegram/ThirdParty/scudo/mem_map.h index 409e4dbbe..b92216cf2 100644 --- a/Telegram/ThirdParty/scudo/mem_map.h +++ b/Telegram/ThirdParty/scudo/mem_map.h @@ -22,6 +22,7 @@ #include "trusty.h" #include "mem_map_fuchsia.h" +#include "mem_map_linux.h" namespace scudo { @@ -73,10 +74,10 @@ private: }; #if SCUDO_LINUX -using ReservedMemoryT = ReservedMemoryDefault; +using ReservedMemoryT = ReservedMemoryLinux; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_FUCHSIA -using ReservedMemoryT = ReservedMemoryDefault; +using ReservedMemoryT = ReservedMemoryFuchsia; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_TRUSTY using ReservedMemoryT = ReservedMemoryDefault; diff --git a/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp b/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp index 9ace1fef7..0566ab065 100644 --- a/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp +++ b/Telegram/ThirdParty/scudo/mem_map_fuchsia.cpp @@ -41,7 +41,7 @@ static void setVmoName(zx_handle_t Vmo, const char *Name) { static uptr getRootVmarBase() { static atomic_uptr CachedResult = {0}; - uptr Result = atomic_load_relaxed(&CachedResult); + uptr Result = atomic_load(&CachedResult, memory_order_acquire); if (UNLIKELY(!Result)) { zx_info_vmar_t VmarInfo; zx_status_t Status = @@ -50,7 +50,7 @@ static uptr getRootVmarBase() { CHECK_EQ(Status, ZX_OK); CHECK_NE(VmarInfo.base, 0); - atomic_store_relaxed(&CachedResult, VmarInfo.base); + atomic_store(&CachedResult, VmarInfo.base, memory_order_release); Result = VmarInfo.base; } @@ -61,7 +61,7 @@ static uptr getRootVmarBase() { static zx_handle_t getPlaceholderVmo() { static atomic_u32 StoredVmo = {ZX_HANDLE_INVALID}; - zx_handle_t Vmo = atomic_load_relaxed(&StoredVmo); + zx_handle_t Vmo = atomic_load(&StoredVmo, memory_order_acquire); if (UNLIKELY(Vmo == ZX_HANDLE_INVALID)) { // Create a zero-sized placeholder VMO. zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); @@ -72,9 +72,9 @@ static zx_handle_t getPlaceholderVmo() { // Atomically store its handle. If some other thread wins the race, use its // handle and discard ours. - zx_handle_t OldValue = - atomic_compare_exchange(&StoredVmo, ZX_HANDLE_INVALID, Vmo); - if (OldValue != ZX_HANDLE_INVALID) { + zx_handle_t OldValue = atomic_compare_exchange_strong( + &StoredVmo, ZX_HANDLE_INVALID, Vmo, memory_order_acq_rel); + if (UNLIKELY(OldValue != ZX_HANDLE_INVALID)) { Status = _zx_handle_close(Vmo); CHECK_EQ(Status, ZX_OK); diff --git a/Telegram/ThirdParty/scudo/mem_map_linux.cpp b/Telegram/ThirdParty/scudo/mem_map_linux.cpp new file mode 100644 index 000000000..783c4f0d9 --- /dev/null +++ b/Telegram/ThirdParty/scudo/mem_map_linux.cpp @@ -0,0 +1,153 @@ +//===-- mem_map_linux.cpp ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX + +#include "mem_map_linux.h" + +#include "common.h" +#include "internal_defs.h" +#include "linux.h" +#include "mutex.h" +#include "report_linux.h" +#include "string_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if SCUDO_ANDROID +// TODO(chiahungduan): Review if we still need the followings macros. +#include +// Definitions of prctl arguments to set a vma name in Android kernels. +#define ANDROID_PR_SET_VMA 0x53564d41 +#define ANDROID_PR_SET_VMA_ANON_NAME 0 +#endif + +namespace scudo { + +static void *mmapWrapper(uptr Addr, uptr Size, const char *Name, uptr Flags) { + int MmapFlags = MAP_PRIVATE | MAP_ANONYMOUS; + int MmapProt; + if (Flags & MAP_NOACCESS) { + MmapFlags |= MAP_NORESERVE; + MmapProt = PROT_NONE; + } else { + MmapProt = PROT_READ | PROT_WRITE; + } +#if defined(__aarch64__) +#ifndef PROT_MTE +#define PROT_MTE 0x20 +#endif + if (Flags & MAP_MEMTAG) + MmapProt |= PROT_MTE; +#endif + if (Addr) + MmapFlags |= MAP_FIXED; + void *P = + mmap(reinterpret_cast(Addr), Size, MmapProt, MmapFlags, -1, 0); + if (P == MAP_FAILED) { + if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM) + reportMapError(errno == ENOMEM ? Size : 0); + return nullptr; + } +#if SCUDO_ANDROID + if (Name) + prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name); +#else + (void)Name; +#endif + + return P; +} + +bool MemMapLinux::mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags) { + void *P = mmapWrapper(Addr, Size, Name, Flags); + if (P == nullptr) + return false; + + MapBase = reinterpret_cast(P); + MapCapacity = Size; + return true; +} + +void MemMapLinux::unmapImpl(uptr Addr, uptr Size) { + // If we unmap all the pages, also mark `MapBase` to 0 to indicate invalid + // status. + if (Size == MapCapacity) { + MapBase = MapCapacity = 0; + } else { + // This is partial unmap and is unmapping the pages from the beginning, + // shift `MapBase` to the new base. + if (MapBase == Addr) + MapBase = Addr + Size; + MapCapacity -= Size; + } + + if (munmap(reinterpret_cast(Addr), Size) != 0) + reportUnmapError(Addr, Size); +} + +bool MemMapLinux::remapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + void *P = mmapWrapper(Addr, Size, Name, Flags); + if (reinterpret_cast(P) != Addr) + reportMapError(); + return true; +} + +void MemMapLinux::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) { + int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE); + if (mprotect(reinterpret_cast(Addr), Size, Prot) != 0) + reportProtectError(Addr, Size, Prot); +} + +void MemMapLinux::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) { + void *Addr = reinterpret_cast(From); + + while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) { + } +} + +bool ReservedMemoryLinux::createImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + ReservedMemoryLinux::MemMapT MemMap; + if (!MemMap.map(Addr, Size, Name, Flags | MAP_NOACCESS)) + return false; + + MapBase = MemMap.getBase(); + MapCapacity = MemMap.getCapacity(); + + return true; +} + +void ReservedMemoryLinux::releaseImpl() { + if (munmap(reinterpret_cast(getBase()), getCapacity()) != 0) + reportUnmapError(getBase(), getCapacity()); +} + +ReservedMemoryLinux::MemMapT ReservedMemoryLinux::dispatchImpl(uptr Addr, + uptr Size) { + return ReservedMemoryLinux::MemMapT(Addr, Size); +} + +} // namespace scudo + +#endif // SCUDO_LINUX diff --git a/Telegram/ThirdParty/scudo/mem_map_linux.h b/Telegram/ThirdParty/scudo/mem_map_linux.h new file mode 100644 index 000000000..7a89b3bff --- /dev/null +++ b/Telegram/ThirdParty/scudo/mem_map_linux.h @@ -0,0 +1,67 @@ +//===-- mem_map_linux.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MEM_MAP_LINUX_H_ +#define SCUDO_MEM_MAP_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX + +#include "common.h" +#include "mem_map_base.h" + +namespace scudo { + +class MemMapLinux final : public MemMapBase { +public: + constexpr MemMapLinux() = default; + MemMapLinux(uptr Base, uptr Capacity) + : MapBase(Base), MapCapacity(Capacity) {} + + // Impls for base functions. + bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void unmapImpl(uptr Addr, uptr Size); + bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags); + void releasePagesToOSImpl(uptr From, uptr Size) { + return releaseAndZeroPagesToOSImpl(From, Size); + } + void releaseAndZeroPagesToOSImpl(uptr From, uptr Size); + uptr getBaseImpl() { return MapBase; } + uptr getCapacityImpl() { return MapCapacity; } + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class ReservedMemoryLinux final + : public ReservedMemory { +public: + // The following two are the Impls for function in `MemMapBase`. + uptr getBaseImpl() { return MapBase; } + uptr getCapacityImpl() { return MapCapacity; } + + // These threes are specific to `ReservedMemory`. + bool createImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void releaseImpl(); + MemMapT dispatchImpl(uptr Addr, uptr Size); + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; +}; + +} // namespace scudo + +#endif // SCUDO_LINUX + +#endif // SCUDO_MEM_MAP_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/mutex.h b/Telegram/ThirdParty/scudo/mutex.h index 05340de3e..4caa94521 100644 --- a/Telegram/ThirdParty/scudo/mutex.h +++ b/Telegram/ThirdParty/scudo/mutex.h @@ -35,7 +35,7 @@ public: #pragma nounroll #endif for (u8 I = 0U; I < NumberOfTries; I++) { - yieldProcessor(NumberOfYields); + delayLoop(); if (tryLock()) return; } @@ -53,10 +53,23 @@ public: } private: + void delayLoop() { + // The value comes from the average time spent in accessing caches (which + // are the fastest operations) so that we are unlikely to wait too long for + // fast operations. + constexpr u32 SpinTimes = 16; + volatile u32 V = 0; + for (u32 I = 0; I < SpinTimes; ++I) { + u32 Tmp = V + 1; + V = Tmp; + } + } + void assertHeldImpl(); - static constexpr u8 NumberOfTries = 8U; - static constexpr u8 NumberOfYields = 8U; + // TODO(chiahungduan): Adapt this value based on scenarios. E.g., primary and + // secondary allocator have different allocation times. + static constexpr u8 NumberOfTries = 32U; #if SCUDO_LINUX atomic_u32 M = {}; diff --git a/Telegram/ThirdParty/scudo/options.h b/Telegram/ThirdParty/scudo/options.h index 4e6786513..b20142a41 100644 --- a/Telegram/ThirdParty/scudo/options.h +++ b/Telegram/ThirdParty/scudo/options.h @@ -38,7 +38,7 @@ struct Options { } }; -template bool useMemoryTagging(Options Options) { +template bool useMemoryTagging(const Options &Options) { return allocatorSupportsMemoryTagging() && Options.get(OptionBit::UseMemoryTagging); } diff --git a/Telegram/ThirdParty/scudo/platform.h b/Telegram/ThirdParty/scudo/platform.h index 7c7024ff5..b71a86be7 100644 --- a/Telegram/ThirdParty/scudo/platform.h +++ b/Telegram/ThirdParty/scudo/platform.h @@ -63,6 +63,20 @@ #define SCUDO_CAN_USE_MTE (SCUDO_LINUX || SCUDO_TRUSTY) #endif +// Use smaller table sizes for fuzzing in order to reduce input size. +// Trusty just has less available memory. +#ifndef SCUDO_SMALL_STACK_DEPOT +#if defined(SCUDO_FUZZ) || SCUDO_TRUSTY +#define SCUDO_SMALL_STACK_DEPOT 1 +#else +#define SCUDO_SMALL_STACK_DEPOT 0 +#endif +#endif + +#ifndef SCUDO_ENABLE_HOOKS +#define SCUDO_ENABLE_HOOKS 0 +#endif + #ifndef SCUDO_MIN_ALIGNMENT_LOG // We force malloc-type functions to be aligned to std::max_align_t, but there // is no reason why the minimum alignment for all other functions can't be 8 diff --git a/Telegram/ThirdParty/scudo/primary32.h b/Telegram/ThirdParty/scudo/primary32.h index 1d8a34ec6..4d03b282d 100644 --- a/Telegram/ThirdParty/scudo/primary32.h +++ b/Telegram/ThirdParty/scudo/primary32.h @@ -9,6 +9,7 @@ #ifndef SCUDO_PRIMARY32_H_ #define SCUDO_PRIMARY32_H_ +#include "allocator_common.h" #include "bytemap.h" #include "common.h" #include "list.h" @@ -53,12 +54,15 @@ public: ""); typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; - typedef typename CacheT::TransferBatch TransferBatch; - typedef typename CacheT::BatchGroup BatchGroup; + typedef TransferBatch TransferBatchT; + typedef BatchGroup BatchGroupT; + + static_assert(sizeof(BatchGroupT) <= sizeof(TransferBatchT), + "BatchGroupT uses the same class size as TransferBatchT"); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) - ? sizeof(TransferBatch) + ? sizeof(TransferBatchT) : SizeClassMap::getSizeByClassId(ClassId); } @@ -126,7 +130,7 @@ public: SizeClassInfo *Sci = getSizeClassInfo(I); ScopedLock L1(Sci->Mutex); uptr TotalBlocks = 0; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`. BatchClassUsedInFreeLists += BG.Batches.size() + 1; for (const auto &It : BG.Batches) @@ -141,7 +145,7 @@ public: SizeClassInfo *Sci = getSizeClassInfo(SizeClassMap::BatchClassId); ScopedLock L1(Sci->Mutex); uptr TotalBlocks = 0; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { if (LIKELY(!BG.Batches.empty())) { for (const auto &It : BG.Batches) TotalBlocks += It.getCount(); @@ -187,11 +191,30 @@ public: return BlockSize > PageSize; } - TransferBatch *popBatch(CacheT *C, uptr ClassId) { + // Note that the `MaxBlockCount` will be used when we support arbitrary blocks + // count. Now it's the same as the number of blocks stored in the + // `TransferBatch`. + u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray, + UNUSED const u16 MaxBlockCount) { + TransferBatchT *B = popBatch(C, ClassId); + if (!B) + return 0; + + const u16 Count = B->getCount(); + DCHECK_GT(Count, 0U); + B->moveToArray(ToArray); + + if (ClassId != SizeClassMap::BatchClassId) + C->deallocate(SizeClassMap::BatchClassId, B); + + return Count; + } + + TransferBatchT *popBatch(CacheT *C, uptr ClassId) { DCHECK_LT(ClassId, NumClasses); SizeClassInfo *Sci = getSizeClassInfo(ClassId); ScopedLock L(Sci->Mutex); - TransferBatch *B = popBatchImpl(C, ClassId, Sci); + TransferBatchT *B = popBatchImpl(C, ClassId, Sci); if (UNLIKELY(!B)) { if (UNLIKELY(!populateFreeList(C, ClassId, Sci))) return nullptr; @@ -311,6 +334,18 @@ public: } } + void getFragmentationInfo(ScopedString *Str) { + Str->append( + "Fragmentation Stats: SizeClassAllocator32: page size = %zu bytes\n", + getPageSizeCached()); + + for (uptr I = 1; I < NumClasses; I++) { + SizeClassInfo *Sci = getSizeClassInfo(I); + ScopedLock L(Sci->Mutex); + getSizeClassFragmentationInfo(Sci, I, Str); + } + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), @@ -369,7 +404,7 @@ private: }; struct BlocksInfo { - SinglyLinkedList BlockList = {}; + SinglyLinkedList BlockList = {}; uptr PoppedBlocks = 0; uptr PushedBlocks = 0; }; @@ -493,11 +528,11 @@ private: // reusable and don't need additional space for them. Sci->FreeListInfo.PushedBlocks += Size; - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); if (BG == nullptr) { // Construct `BatchGroup` on the last element. - BG = reinterpret_cast( + BG = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); --Size; BG->Batches.clear(); @@ -508,8 +543,8 @@ private: // from `CreateGroup` in `pushBlocksImpl` BG->PushedBlocks = 1; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = TransferBatch::getMaxCached( - getSizeByClassId(SizeClassMap::BatchClassId)); + BG->MaxCachedPerBatch = + CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId)); Sci->FreeListInfo.BlockList.push_front(BG); } @@ -522,7 +557,7 @@ private: // 2. Only 1 block is pushed when the freelist is empty. if (BG->Batches.empty()) { // Construct the `TransferBatch` on the last element. - TransferBatch *TB = reinterpret_cast( + TransferBatchT *TB = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); TB->clear(); // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are @@ -537,14 +572,14 @@ private: BG->Batches.push_front(TB); } - TransferBatch *CurBatch = BG->Batches.front(); + TransferBatchT *CurBatch = BG->Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = reinterpret_cast( + CurBatch = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[I])); CurBatch->clear(); // Self-contained @@ -588,24 +623,25 @@ private: DCHECK_GT(Size, 0U); auto CreateGroup = [&](uptr CompactPtrGroupBase) { - BatchGroup *BG = C->createGroup(); + BatchGroupT *BG = + reinterpret_cast(C->getBatchClassBlock()); BG->Batches.clear(); - TransferBatch *TB = C->createBatch(ClassId, nullptr); + TransferBatchT *TB = + reinterpret_cast(C->getBatchClassBlock()); TB->clear(); BG->CompactPtrGroupBase = CompactPtrGroupBase; BG->Batches.push_front(TB); BG->PushedBlocks = 0; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = - TransferBatch::getMaxCached(getSizeByClassId(ClassId)); + BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId)); return BG; }; - auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) { - SinglyLinkedList &Batches = BG->Batches; - TransferBatch *CurBatch = Batches.front(); + auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) { + SinglyLinkedList &Batches = BG->Batches; + TransferBatchT *CurBatch = Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { @@ -613,9 +649,8 @@ private: u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = C->createBatch( - ClassId, - reinterpret_cast(decompactPtr(ClassId, Array[I]))); + CurBatch = + reinterpret_cast(C->getBatchClassBlock()); CurBatch->clear(); Batches.push_front(CurBatch); UnusedSlots = BG->MaxCachedPerBatch; @@ -630,11 +665,11 @@ private: }; Sci->FreeListInfo.PushedBlocks += Size; - BatchGroup *Cur = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *Cur = Sci->FreeListInfo.BlockList.front(); // In the following, `Cur` always points to the BatchGroup for blocks that // will be pushed next. `Prev` is the element right before `Cur`. - BatchGroup *Prev = nullptr; + BatchGroupT *Prev = nullptr; while (Cur != nullptr && compactPtrGroupBase(Array[0]) > Cur->CompactPtrGroupBase) { @@ -695,22 +730,22 @@ private: // group id will be considered first. // // The region mutex needs to be held while calling this method. - TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci) + TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci) REQUIRES(Sci->Mutex) { if (Sci->FreeListInfo.BlockList.empty()) return nullptr; - SinglyLinkedList &Batches = + SinglyLinkedList &Batches = Sci->FreeListInfo.BlockList.front()->Batches; if (Batches.empty()) { DCHECK_EQ(ClassId, SizeClassMap::BatchClassId); - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); Sci->FreeListInfo.BlockList.pop_front(); // Block used by `BatchGroup` is from BatchClassId. Turn the block into // `TransferBatch` with single block. - TransferBatch *TB = reinterpret_cast(BG); + TransferBatchT *TB = reinterpret_cast(BG); TB->clear(); TB->add( compactPtr(SizeClassMap::BatchClassId, reinterpret_cast(TB))); @@ -718,13 +753,13 @@ private: return TB; } - TransferBatch *B = Batches.front(); + TransferBatchT *B = Batches.front(); Batches.pop_front(); DCHECK_NE(B, nullptr); DCHECK_GT(B->getCount(), 0U); if (Batches.empty()) { - BatchGroup *BG = Sci->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Sci->FreeListInfo.BlockList.front(); Sci->FreeListInfo.BlockList.pop_front(); // We don't keep BatchGroup with zero blocks to avoid empty-checking while @@ -763,7 +798,7 @@ private: } const uptr Size = getSizeByClassId(ClassId); - const u16 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = CacheT::getMaxCached(Size); DCHECK_GT(MaxCount, 0U); // The maximum number of blocks we should carve in the region is dictated // by the maximum number of batches we want to fill, and the amount of @@ -776,7 +811,7 @@ private: DCHECK_GT(NumberOfBlocks, 0U); constexpr u32 ShuffleArraySize = - MaxNumBatches * TransferBatch::MaxNumCached; + MaxNumBatches * TransferBatchT::MaxNumCached; // Fill the transfer batches and put them in the size-class freelist. We // need to randomize the blocks for security purposes, so we first fill a // local array that we then shuffle before populating the batches. @@ -856,11 +891,60 @@ private: PushedBytesDelta >> 10); } + void getSizeClassFragmentationInfo(SizeClassInfo *Sci, uptr ClassId, + ScopedString *Str) REQUIRES(Sci->Mutex) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr First = Sci->MinRegionIndex; + const uptr Last = Sci->MaxRegionIndex; + const uptr Base = First * RegionSize; + const uptr NumberOfRegions = Last - First + 1U; + auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { + ScopedLock L(ByteMapMutex); + return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; + }; + + FragmentationRecorder Recorder; + if (!Sci->FreeListInfo.BlockList.empty()) { + PageReleaseContext Context = + markFreeBlocks(Sci, ClassId, BlockSize, Base, NumberOfRegions, + ReleaseToOS::ForceAll); + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + } + + const uptr PageSize = getPageSizeCached(); + const uptr TotalBlocks = Sci->AllocatedUser / BlockSize; + const uptr InUseBlocks = + Sci->FreeListInfo.PoppedBlocks - Sci->FreeListInfo.PushedBlocks; + uptr AllocatedPagesCount = 0; + if (TotalBlocks != 0U) { + for (uptr I = 0; I < NumberOfRegions; ++I) { + if (SkipRegion(I)) + continue; + AllocatedPagesCount += RegionSize / PageSize; + } + + DCHECK_NE(AllocatedPagesCount, 0U); + } + + DCHECK_GE(AllocatedPagesCount, Recorder.getReleasedPagesCount()); + const uptr InUsePages = + AllocatedPagesCount - Recorder.getReleasedPagesCount(); + const uptr InUseBytes = InUsePages * PageSize; + + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); + Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", + ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); + } + NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId, ReleaseToOS ReleaseType = ReleaseToOS::Normal) REQUIRES(Sci->Mutex) { const uptr BlockSize = getSizeByClassId(ClassId); - const uptr PageSize = getPageSizeCached(); DCHECK_GE(Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks); const uptr BytesInFreeList = @@ -871,6 +955,60 @@ private: if (UNLIKELY(BytesInFreeList == 0)) return 0; + // ====================================================================== // + // 1. Check if we have enough free blocks and if it's worth doing a page + // release. + // ====================================================================== // + if (ReleaseType != ReleaseToOS::ForceAll && + !hasChanceToReleasePages(Sci, BlockSize, BytesInFreeList, + ReleaseType)) { + return 0; + } + + const uptr First = Sci->MinRegionIndex; + const uptr Last = Sci->MaxRegionIndex; + DCHECK_NE(Last, 0U); + DCHECK_LE(First, Last); + uptr TotalReleasedBytes = 0; + const uptr Base = First * RegionSize; + const uptr NumberOfRegions = Last - First + 1U; + + // ==================================================================== // + // 2. Mark the free blocks and we can tell which pages are in-use by + // querying `PageReleaseContext`. + // ==================================================================== // + PageReleaseContext Context = markFreeBlocks(Sci, ClassId, BlockSize, Base, + NumberOfRegions, ReleaseType); + if (!Context.hasBlockMarked()) + return 0; + + // ==================================================================== // + // 3. Release the unused physical pages back to the OS. + // ==================================================================== // + ReleaseRecorder Recorder(Base); + auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { + ScopedLock L(ByteMapMutex); + return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; + }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + + if (Recorder.getReleasedRangesCount() > 0) { + Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; + Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes; + } + Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); + + return TotalReleasedBytes; + } + + bool hasChanceToReleasePages(SizeClassInfo *Sci, uptr BlockSize, + uptr BytesInFreeList, ReleaseToOS ReleaseType) + REQUIRES(Sci->Mutex) { + DCHECK_GE(Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks); + const uptr PageSize = getPageSizeCached(); + if (BytesInFreeList <= Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint) Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; @@ -892,22 +1030,20 @@ private: // (BytesInFreeListAtLastCheckpoint - BytesInFreeList). const uptr PushedBytesDelta = BytesInFreeList - Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint; - if (PushedBytesDelta < PageSize && ReleaseType != ReleaseToOS::ForceAll) - return 0; + if (PushedBytesDelta < PageSize) + return false; - const bool CheckDensity = - isSmallBlock(BlockSize) && ReleaseType != ReleaseToOS::ForceAll; // Releasing smaller blocks is expensive, so we want to make sure that a // significant amount of bytes are free, and that there has been a good // amount of batches pushed to the freelist before attempting to release. - if (CheckDensity && ReleaseType == ReleaseToOS::Normal) + if (isSmallBlock(BlockSize) && ReleaseType == ReleaseToOS::Normal) if (PushedBytesDelta < Sci->AllocatedUser / 16U) - return 0; + return false; if (ReleaseType == ReleaseToOS::Normal) { const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs); if (IntervalMs < 0) - return 0; + return false; // The constant 8 here is selected from profiling some apps and the number // of unreleased pages in the large size classes is around 16 pages or @@ -920,30 +1056,31 @@ private: static_cast(IntervalMs) * 1000000 > getMonotonicTimeFast()) { // Memory was returned recently. - return 0; + return false; } } } // if (ReleaseType == ReleaseToOS::Normal) - const uptr First = Sci->MinRegionIndex; - const uptr Last = Sci->MaxRegionIndex; - DCHECK_NE(Last, 0U); - DCHECK_LE(First, Last); - uptr TotalReleasedBytes = 0; - const uptr Base = First * RegionSize; - const uptr NumberOfRegions = Last - First + 1U; - const uptr GroupSize = (1U << GroupSizeLog); + return true; + } + + PageReleaseContext markFreeBlocks(SizeClassInfo *Sci, const uptr ClassId, + const uptr BlockSize, const uptr Base, + const uptr NumberOfRegions, + ReleaseToOS ReleaseType) + REQUIRES(Sci->Mutex) { + const uptr PageSize = getPageSizeCached(); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr CurGroupBase = compactPtrGroupBase(compactPtr(ClassId, Sci->CurrentRegion)); - ReleaseRecorder Recorder(Base); PageReleaseContext Context(BlockSize, NumberOfRegions, /*ReleaseSize=*/RegionSize); auto DecompactPtr = [](CompactPtrT CompactPtr) { return reinterpret_cast(CompactPtr); }; - for (BatchGroup &BG : Sci->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Sci->FreeListInfo.BlockList) { const uptr GroupBase = decompactGroupBase(BG.CompactPtrGroupBase); // The `GroupSize` may not be divided by `BlockSize`, which means there is // an unused space at the end of Region. Exclude that space to avoid @@ -960,25 +1097,27 @@ private: BG.Batches.front()->getCount(); const uptr BytesInBG = NumBlocks * BlockSize; - if (ReleaseType != ReleaseToOS::ForceAll && - BytesInBG <= BG.BytesInBGAtLastCheckpoint) { - BG.BytesInBGAtLastCheckpoint = BytesInBG; - continue; - } - const uptr PushedBytesDelta = BytesInBG - BG.BytesInBGAtLastCheckpoint; - if (ReleaseType != ReleaseToOS::ForceAll && PushedBytesDelta < PageSize) - continue; + if (ReleaseType != ReleaseToOS::ForceAll) { + if (BytesInBG <= BG.BytesInBGAtLastCheckpoint) { + BG.BytesInBGAtLastCheckpoint = BytesInBG; + continue; + } - // Given the randomness property, we try to release the pages only if the - // bytes used by free blocks exceed certain proportion of allocated - // spaces. - if (CheckDensity && (BytesInBG * 100U) / AllocatedGroupSize < - (100U - 1U - BlockSize / 16U)) { - continue; + const uptr PushedBytesDelta = BytesInBG - BG.BytesInBGAtLastCheckpoint; + if (PushedBytesDelta < PageSize) + continue; + + // Given the randomness property, we try to release the pages only if + // the bytes used by free blocks exceed certain proportion of allocated + // spaces. + if (isSmallBlock(BlockSize) && (BytesInBG * 100U) / AllocatedGroupSize < + (100U - 1U - BlockSize / 16U)) { + continue; + } } // TODO: Consider updating this after page release if `ReleaseRecorder` - // can tell the releasd bytes in each group. + // can tell the released bytes in each group. BG.BytesInBGAtLastCheckpoint = BytesInBG; const uptr MaxContainedBlocks = AllocatedGroupSize / BlockSize; @@ -1006,27 +1145,10 @@ private: // We may not be able to do the page release In a rare case that we may // fail on PageMap allocation. if (UNLIKELY(!Context.hasBlockMarked())) - return 0; + break; } - if (!Context.hasBlockMarked()) - return 0; - - auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { - ScopedLock L(ByteMapMutex); - return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; - }; - releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - - if (Recorder.getReleasedRangesCount() > 0) { - Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); - Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); - TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes; - } - Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); - - return TotalReleasedBytes; + return Context; } SizeClassInfo SizeClassInfoArray[NumClasses] = {}; diff --git a/Telegram/ThirdParty/scudo/primary64.h b/Telegram/ThirdParty/scudo/primary64.h index fd7a1f9e8..9a642d236 100644 --- a/Telegram/ThirdParty/scudo/primary64.h +++ b/Telegram/ThirdParty/scudo/primary64.h @@ -9,6 +9,7 @@ #ifndef SCUDO_PRIMARY64_H_ #define SCUDO_PRIMARY64_H_ +#include "allocator_common.h" #include "bytemap.h" #include "common.h" #include "list.h" @@ -21,6 +22,8 @@ #include "string_utils.h" #include "thread_annotations.h" +#include "condition_variable.h" + namespace scudo { // SizeClassAllocator64 is an allocator tuned for 64-bit address space. @@ -47,27 +50,39 @@ template class SizeClassAllocator64 { public: typedef typename Config::Primary::CompactPtrT CompactPtrT; typedef typename Config::Primary::SizeClassMap SizeClassMap; + typedef typename ConditionVariableState< + typename Config::Primary>::ConditionVariableT ConditionVariableT; static const uptr CompactPtrScale = Config::Primary::CompactPtrScale; + static const uptr RegionSizeLog = Config::Primary::RegionSizeLog; static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; + static_assert(RegionSizeLog >= GroupSizeLog, + "Group size shouldn't be greater than the region size"); static const uptr GroupScale = GroupSizeLog - CompactPtrScale; typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; - typedef typename CacheT::TransferBatch TransferBatch; - typedef typename CacheT::BatchGroup BatchGroup; + typedef TransferBatch TransferBatchT; + typedef BatchGroup BatchGroupT; + + static_assert(sizeof(BatchGroupT) <= sizeof(TransferBatchT), + "BatchGroupT uses the same class size as TransferBatchT"); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) - ? roundUp(sizeof(TransferBatch), 1U << CompactPtrScale) + ? roundUp(sizeof(TransferBatchT), 1U << CompactPtrScale) : SizeClassMap::getSizeByClassId(ClassId); } static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } + static bool conditionVariableEnabled() { + return ConditionVariableState::enabled(); + } + void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS { DCHECK(isAligned(reinterpret_cast(this), alignof(ThisT))); const uptr PageSize = getPageSizeCached(); - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr PagesInGroup = GroupSize / PageSize; const uptr MinSizeClass = getSizeByClassId(1); // When trying to release pages back to memory, visiting smaller size @@ -117,13 +132,13 @@ public: for (uptr I = 0; I < NumClasses; I++) { RegionInfo *Region = getRegionInfo(I); + // The actual start of a region is offset by a random number of pages // when PrimaryEnableRandomOffset is set. - Region->RegionBeg = - (PrimaryBase + (I << Config::Primary::RegionSizeLog)) + - (Config::Primary::EnableRandomOffset - ? ((getRandomModN(&Seed, 16) + 1) * PageSize) - : 0); + Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) + + (Config::Primary::EnableRandomOffset + ? ((getRandomModN(&Seed, 16) + 1) * PageSize) + : 0); Region->RandState = getRandomU32(&Seed); // Releasing small blocks is expensive, set a higher threshold to avoid // frequent page releases. @@ -134,11 +149,16 @@ public: Region->ReleaseInfo.LastReleaseAtNs = Time; Region->MemMapInfo.MemMap = ReservedMemory.dispatch( - PrimaryBase + (I << Config::Primary::RegionSizeLog), RegionSize); + PrimaryBase + (I << RegionSizeLog), RegionSize); CHECK(Region->MemMapInfo.MemMap.isAllocated()); } shuffle(RegionInfoArray, NumClasses, &Seed); + // The binding should be done after region shuffling so that it won't bind + // the FLLock from the wrong region. + for (uptr I = 0; I < NumClasses; I++) + getRegionInfo(I)->FLLockCV.bindTestOnly(getRegionInfo(I)->FLLock); + setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } @@ -165,7 +185,7 @@ public: ScopedLock FL(Region->FLLock); const uptr BlockSize = getSizeByClassId(I); uptr TotalBlocks = 0; - for (BatchGroup &BG : Region->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { // `BG::Batches` are `TransferBatches`. +1 for `BatchGroup`. BatchClassUsedInFreeLists += BG.Batches.size() + 1; for (const auto &It : BG.Batches) @@ -182,7 +202,7 @@ public: ScopedLock FL(Region->FLLock); const uptr BlockSize = getSizeByClassId(SizeClassMap::BatchClassId); uptr TotalBlocks = 0; - for (BatchGroup &BG : Region->FreeListInfo.BlockList) { + for (BatchGroupT &BG : Region->FreeListInfo.BlockList) { if (LIKELY(!BG.Batches.empty())) { for (const auto &It : BG.Batches) TotalBlocks += It.getCount(); @@ -201,51 +221,64 @@ public: DCHECK_EQ(BlocksInUse, BatchClassUsedInFreeLists); } - TransferBatch *popBatch(CacheT *C, uptr ClassId) { + // Note that the `MaxBlockCount` will be used when we support arbitrary blocks + // count. Now it's the same as the number of blocks stored in the + // `TransferBatch`. + u16 popBlocks(CacheT *C, uptr ClassId, CompactPtrT *ToArray, + UNUSED const u16 MaxBlockCount) { + TransferBatchT *B = popBatch(C, ClassId); + if (!B) + return 0; + + const u16 Count = B->getCount(); + DCHECK_GT(Count, 0U); + B->moveToArray(ToArray); + + if (ClassId != SizeClassMap::BatchClassId) + C->deallocate(SizeClassMap::BatchClassId, B); + + return Count; + } + + TransferBatchT *popBatch(CacheT *C, uptr ClassId) { DCHECK_LT(ClassId, NumClasses); RegionInfo *Region = getRegionInfo(ClassId); { ScopedLock L(Region->FLLock); - TransferBatch *B = popBatchImpl(C, ClassId, Region); + TransferBatchT *B = popBatchImpl(C, ClassId, Region); if (LIKELY(B)) return B; } - bool PrintStats = false; - TransferBatch *B = nullptr; + bool ReportRegionExhausted = false; + TransferBatchT *B = nullptr; - while (true) { - // When two threads compete for `Region->MMLock`, we only want one of them - // does the populateFreeListAndPopBatch(). To avoid both of them doing - // that, always check the freelist before mapping new pages. - // - // TODO(chiahungduan): Use a condition variable so that we don't need to - // hold `Region->MMLock` here. - ScopedLock ML(Region->MMLock); - { - ScopedLock FL(Region->FLLock); - B = popBatchImpl(C, ClassId, Region); - if (LIKELY(B)) - return B; + if (conditionVariableEnabled()) { + B = popBatchWithCV(C, ClassId, Region, ReportRegionExhausted); + } else { + while (true) { + // When two threads compete for `Region->MMLock`, we only want one of + // them to call populateFreeListAndPopBatch(). To avoid both of them + // doing that, always check the freelist before mapping new pages. + ScopedLock ML(Region->MMLock); + { + ScopedLock FL(Region->FLLock); + if ((B = popBatchImpl(C, ClassId, Region))) + break; + } + + const bool RegionIsExhausted = Region->Exhausted; + if (!RegionIsExhausted) + B = populateFreeListAndPopBatch(C, ClassId, Region); + ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; + break; } - - const bool RegionIsExhausted = Region->Exhausted; - if (!RegionIsExhausted) - B = populateFreeListAndPopBatch(C, ClassId, Region); - PrintStats = !RegionIsExhausted && Region->Exhausted; - break; } - // Note that `getStats()` requires locking each region so we can't call it - // while locking the Region->Mutex in the above. - if (UNLIKELY(PrintStats)) { - ScopedString Str; - getStats(&Str); - Str.append( - "Scudo OOM: The process has exhausted %zuM for size class %zu.\n", - RegionSize >> 20, getSizeByClassId(ClassId)); - Str.output(); + if (UNLIKELY(ReportRegionExhausted)) { + Printf("Can't populate more pages for size class %zu.\n", + getSizeByClassId(ClassId)); // Theoretically, BatchClass shouldn't be used up. Abort immediately when // it happens. @@ -265,30 +298,36 @@ public: if (ClassId == SizeClassMap::BatchClassId) { ScopedLock L(Region->FLLock); pushBatchClassBlocks(Region, Array, Size); + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); return; } // TODO(chiahungduan): Consider not doing grouping if the group size is not // greater than the block size with a certain scale. - // Sort the blocks so that blocks belonging to the same group can be pushed - // together. bool SameGroup = true; - for (u32 I = 1; I < Size; ++I) { - if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) - SameGroup = false; - CompactPtrT Cur = Array[I]; - u32 J = I; - while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) { - Array[J] = Array[J - 1]; - --J; + if (GroupSizeLog < RegionSizeLog) { + // Sort the blocks so that blocks belonging to the same group can be + // pushed together. + for (u32 I = 1; I < Size; ++I) { + if (compactPtrGroup(Array[I - 1]) != compactPtrGroup(Array[I])) + SameGroup = false; + CompactPtrT Cur = Array[I]; + u32 J = I; + while (J > 0 && compactPtrGroup(Cur) < compactPtrGroup(Array[J - 1])) { + Array[J] = Array[J - 1]; + --J; + } + Array[J] = Cur; } - Array[J] = Cur; } { ScopedLock L(Region->FLLock); pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup); + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); } } @@ -363,6 +402,18 @@ public: } } + void getFragmentationInfo(ScopedString *Str) { + Str->append( + "Fragmentation Stats: SizeClassAllocator64: page size = %zu bytes\n", + getPageSizeCached()); + + for (uptr I = 1; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + ScopedLock L(Region->MMLock); + getRegionFragmentationInfo(Region, I, Str); + } + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), @@ -477,7 +528,7 @@ public: AtomicOptions Options; private: - static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog; + static const uptr RegionSize = 1UL << RegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; @@ -493,7 +544,7 @@ private: }; struct BlocksInfo { - SinglyLinkedList BlockList = {}; + SinglyLinkedList BlockList = {}; uptr PoppedBlocks = 0; uptr PushedBlocks = 0; }; @@ -509,6 +560,7 @@ private: struct UnpaddedRegionInfo { // Mutex for operations on freelist HybridMutex FLLock; + ConditionVariableT FLLockCV GUARDED_BY(FLLock); // Mutex for memmap operations HybridMutex MMLock ACQUIRED_BEFORE(FLLock); // `RegionBeg` is initialized before thread creation and won't be changed. @@ -520,6 +572,7 @@ private: uptr TryReleaseThreshold GUARDED_BY(MMLock) = 0; ReleaseToOsInfo ReleaseInfo GUARDED_BY(MMLock) = {}; bool Exhausted GUARDED_BY(MMLock) = false; + bool isPopulatingFreeList GUARDED_BY(FLLock) = false; }; struct RegionInfo : UnpaddedRegionInfo { char Padding[SCUDO_CACHE_LINE_SIZE - @@ -605,11 +658,11 @@ private: // reusable and don't need additional space for them. Region->FreeListInfo.PushedBlocks += Size; - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); if (BG == nullptr) { // Construct `BatchGroup` on the last element. - BG = reinterpret_cast( + BG = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); --Size; BG->Batches.clear(); @@ -620,8 +673,8 @@ private: // from `CreateGroup` in `pushBlocksImpl` BG->PushedBlocks = 1; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = TransferBatch::getMaxCached( - getSizeByClassId(SizeClassMap::BatchClassId)); + BG->MaxCachedPerBatch = + CacheT::getMaxCached(getSizeByClassId(SizeClassMap::BatchClassId)); Region->FreeListInfo.BlockList.push_front(BG); } @@ -634,7 +687,7 @@ private: // 2. Only 1 block is pushed when the freelist is empty. if (BG->Batches.empty()) { // Construct the `TransferBatch` on the last element. - TransferBatch *TB = reinterpret_cast( + TransferBatchT *TB = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[Size - 1])); TB->clear(); // As mentioned above, addresses of `TransferBatch` and `BatchGroup` are @@ -649,14 +702,14 @@ private: BG->Batches.push_front(TB); } - TransferBatch *CurBatch = BG->Batches.front(); + TransferBatchT *CurBatch = BG->Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = reinterpret_cast( + CurBatch = reinterpret_cast( decompactPtr(SizeClassMap::BatchClassId, Array[I])); CurBatch->clear(); // Self-contained @@ -699,24 +752,25 @@ private: DCHECK_GT(Size, 0U); auto CreateGroup = [&](uptr CompactPtrGroupBase) { - BatchGroup *BG = C->createGroup(); + BatchGroupT *BG = + reinterpret_cast(C->getBatchClassBlock()); BG->Batches.clear(); - TransferBatch *TB = C->createBatch(ClassId, nullptr); + TransferBatchT *TB = + reinterpret_cast(C->getBatchClassBlock()); TB->clear(); BG->CompactPtrGroupBase = CompactPtrGroupBase; BG->Batches.push_front(TB); BG->PushedBlocks = 0; BG->BytesInBGAtLastCheckpoint = 0; - BG->MaxCachedPerBatch = - TransferBatch::getMaxCached(getSizeByClassId(ClassId)); + BG->MaxCachedPerBatch = CacheT::getMaxCached(getSizeByClassId(ClassId)); return BG; }; - auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) { - SinglyLinkedList &Batches = BG->Batches; - TransferBatch *CurBatch = Batches.front(); + auto InsertBlocks = [&](BatchGroupT *BG, CompactPtrT *Array, u32 Size) { + SinglyLinkedList &Batches = BG->Batches; + TransferBatchT *CurBatch = Batches.front(); DCHECK_NE(CurBatch, nullptr); for (u32 I = 0; I < Size;) { @@ -724,9 +778,8 @@ private: u16 UnusedSlots = static_cast(BG->MaxCachedPerBatch - CurBatch->getCount()); if (UnusedSlots == 0) { - CurBatch = C->createBatch( - ClassId, - reinterpret_cast(decompactPtr(ClassId, Array[I]))); + CurBatch = + reinterpret_cast(C->getBatchClassBlock()); CurBatch->clear(); Batches.push_front(CurBatch); UnusedSlots = BG->MaxCachedPerBatch; @@ -741,21 +794,11 @@ private: }; Region->FreeListInfo.PushedBlocks += Size; - BatchGroup *Cur = Region->FreeListInfo.BlockList.front(); - - if (ClassId == SizeClassMap::BatchClassId) { - if (Cur == nullptr) { - // Don't need to classify BatchClassId. - Cur = CreateGroup(/*CompactPtrGroupBase=*/0); - Region->FreeListInfo.BlockList.push_front(Cur); - } - InsertBlocks(Cur, Array, Size); - return; - } + BatchGroupT *Cur = Region->FreeListInfo.BlockList.front(); // In the following, `Cur` always points to the BatchGroup for blocks that // will be pushed next. `Prev` is the element right before `Cur`. - BatchGroup *Prev = nullptr; + BatchGroupT *Prev = nullptr; while (Cur != nullptr && compactPtrGroup(Array[0]) > Cur->CompactPtrGroupBase) { @@ -812,26 +855,96 @@ private: InsertBlocks(Cur, Array + Size - Count, Count); } + TransferBatchT *popBatchWithCV(CacheT *C, uptr ClassId, RegionInfo *Region, + bool &ReportRegionExhausted) { + TransferBatchT *B = nullptr; + + while (true) { + // We only expect one thread doing the freelist refillment and other + // threads will be waiting for either the completion of the + // `populateFreeListAndPopBatch()` or `pushBlocks()` called by other + // threads. + bool PopulateFreeList = false; + { + ScopedLock FL(Region->FLLock); + if (!Region->isPopulatingFreeList) { + Region->isPopulatingFreeList = true; + PopulateFreeList = true; + } + } + + if (PopulateFreeList) { + ScopedLock ML(Region->MMLock); + + const bool RegionIsExhausted = Region->Exhausted; + if (!RegionIsExhausted) + B = populateFreeListAndPopBatch(C, ClassId, Region); + ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted; + + { + // Before reacquiring the `FLLock`, the freelist may be used up again + // and some threads are waiting for the freelist refillment by the + // current thread. It's important to set + // `Region->isPopulatingFreeList` to false so the threads about to + // sleep will notice the status change. + ScopedLock FL(Region->FLLock); + Region->isPopulatingFreeList = false; + Region->FLLockCV.notifyAll(Region->FLLock); + } + + break; + } + + // At here, there are two preconditions to be met before waiting, + // 1. The freelist is empty. + // 2. Region->isPopulatingFreeList == true, i.e, someone is still doing + // `populateFreeListAndPopBatch()`. + // + // Note that it has the chance that freelist is empty but + // Region->isPopulatingFreeList == false because all the new populated + // blocks were used up right after the refillment. Therefore, we have to + // check if someone is still populating the freelist. + ScopedLock FL(Region->FLLock); + if (LIKELY(B = popBatchImpl(C, ClassId, Region))) + break; + + if (!Region->isPopulatingFreeList) + continue; + + // Now the freelist is empty and someone's doing the refillment. We will + // wait until anyone refills the freelist or someone finishes doing + // `populateFreeListAndPopBatch()`. The refillment can be done by + // `populateFreeListAndPopBatch()`, `pushBlocks()`, + // `pushBatchClassBlocks()` and `mergeGroupsToReleaseBack()`. + Region->FLLockCV.wait(Region->FLLock); + + if (LIKELY(B = popBatchImpl(C, ClassId, Region))) + break; + } + + return B; + } + // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest // group id will be considered first. // // The region mutex needs to be held while calling this method. - TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region) + TransferBatchT *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region) REQUIRES(Region->FLLock) { if (Region->FreeListInfo.BlockList.empty()) return nullptr; - SinglyLinkedList &Batches = + SinglyLinkedList &Batches = Region->FreeListInfo.BlockList.front()->Batches; if (Batches.empty()) { DCHECK_EQ(ClassId, SizeClassMap::BatchClassId); - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); Region->FreeListInfo.BlockList.pop_front(); // Block used by `BatchGroup` is from BatchClassId. Turn the block into // `TransferBatch` with single block. - TransferBatch *TB = reinterpret_cast(BG); + TransferBatchT *TB = reinterpret_cast(BG); TB->clear(); TB->add( compactPtr(SizeClassMap::BatchClassId, reinterpret_cast(TB))); @@ -839,13 +952,13 @@ private: return TB; } - TransferBatch *B = Batches.front(); + TransferBatchT *B = Batches.front(); Batches.pop_front(); DCHECK_NE(B, nullptr); DCHECK_GT(B->getCount(), 0U); if (Batches.empty()) { - BatchGroup *BG = Region->FreeListInfo.BlockList.front(); + BatchGroupT *BG = Region->FreeListInfo.BlockList.front(); Region->FreeListInfo.BlockList.pop_front(); // We don't keep BatchGroup with zero blocks to avoid empty-checking while @@ -863,11 +976,11 @@ private: } // Refill the freelist and return one batch. - NOINLINE TransferBatch *populateFreeListAndPopBatch(CacheT *C, uptr ClassId, - RegionInfo *Region) + NOINLINE TransferBatchT *populateFreeListAndPopBatch(CacheT *C, uptr ClassId, + RegionInfo *Region) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { const uptr Size = getSizeByClassId(ClassId); - const u16 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = CacheT::getMaxCached(Size); const uptr RegionBeg = Region->RegionBeg; const uptr MappedUser = Region->MemMapInfo.MappedUser; @@ -903,7 +1016,7 @@ private: DCHECK_GT(NumberOfBlocks, 0); constexpr u32 ShuffleArraySize = - MaxNumBatches * TransferBatch::MaxNumCached; + MaxNumBatches * TransferBatchT::MaxNumCached; CompactPtrT ShuffleArray[ShuffleArraySize]; DCHECK_LE(NumberOfBlocks, ShuffleArraySize); @@ -936,7 +1049,7 @@ private: pushBatchClassBlocks(Region, ShuffleArray, NumberOfBlocks); } - TransferBatch *B = popBatchImpl(C, ClassId, Region); + TransferBatchT *B = popBatchImpl(C, ClassId, Region); DCHECK_NE(B, nullptr); // Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record @@ -957,10 +1070,10 @@ private: if (Region->MemMapInfo.MappedUser == 0) return; const uptr BlockSize = getSizeByClassId(ClassId); - const uptr InUse = + const uptr InUseBlocks = Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; const uptr BytesInFreeList = - Region->MemMapInfo.AllocatedUser - InUse * BlockSize; + Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize; uptr RegionPushedBytesDelta = 0; if (BytesInFreeList >= Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) { @@ -972,122 +1085,145 @@ private: "%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " "inuse: %6zu total: %6zu releases: %6zu last " "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n", - Region->Exhausted ? "F" : " ", ClassId, getSizeByClassId(ClassId), + Region->Exhausted ? "E" : " ", ClassId, getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, - Region->FreeListInfo.PushedBlocks, InUse, TotalChunks, + Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, Region->ReleaseInfo.RangesReleased, Region->ReleaseInfo.LastReleasedBytes >> 10, RegionPushedBytesDelta >> 10, Region->RegionBeg, getRegionBaseByClassId(ClassId)); } + void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, + ScopedString *Str) REQUIRES(Region->MMLock) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr AllocatedUserEnd = + Region->MemMapInfo.AllocatedUser + Region->RegionBeg; + + SinglyLinkedList GroupsToRelease; + { + ScopedLock L(Region->FLLock); + GroupsToRelease = Region->FreeListInfo.BlockList; + Region->FreeListInfo.BlockList.clear(); + } + + FragmentationRecorder Recorder; + if (!GroupsToRelease.empty()) { + PageReleaseContext Context = + markFreeBlocks(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId), GroupsToRelease); + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + + mergeGroupsToReleaseBack(Region, GroupsToRelease); + } + + ScopedLock L(Region->FLLock); + const uptr PageSize = getPageSizeCached(); + const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize; + const uptr InUseBlocks = + Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; + const uptr AllocatedPagesCount = + roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize; + DCHECK_GE(AllocatedPagesCount, Recorder.getReleasedPagesCount()); + const uptr InUsePages = + AllocatedPagesCount - Recorder.getReleasedPagesCount(); + const uptr InUseBytes = InUsePages * PageSize; + + uptr Integral; + uptr Fractional; + computePercentage(BlockSize * InUseBlocks, InUsePages * PageSize, &Integral, + &Fractional); + Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " + "pages: %6zu/%6zu inuse bytes: %6zuK util: %3zu.%02zu%%\n", + ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, + AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional); + } + NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, ReleaseToOS ReleaseType = ReleaseToOS::Normal) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { - ScopedLock L(Region->FLLock); - const uptr BlockSize = getSizeByClassId(ClassId); - const uptr BytesInFreeList = - Region->MemMapInfo.AllocatedUser - (Region->FreeListInfo.PoppedBlocks - - Region->FreeListInfo.PushedBlocks) * - BlockSize; - if (UNLIKELY(BytesInFreeList == 0)) - return false; - + uptr BytesInFreeList; const uptr AllocatedUserEnd = Region->MemMapInfo.AllocatedUser + Region->RegionBeg; - const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId); + SinglyLinkedList GroupsToRelease; - // ====================================================================== // - // 1. Check if we have enough free blocks and if it's worth doing a page - // release. - // ====================================================================== // - if (ReleaseType != ReleaseToOS::ForceAll && - !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList, - ReleaseType)) { - return 0; - } + { + ScopedLock L(Region->FLLock); - // ====================================================================== // - // 2. Determine which groups can release the pages. Use a heuristic to - // gather groups that are candidates for doing a release. - // ====================================================================== // - SinglyLinkedList GroupsToRelease; - if (ReleaseType == ReleaseToOS::ForceAll) { - GroupsToRelease = Region->FreeListInfo.BlockList; - Region->FreeListInfo.BlockList.clear(); - } else { - GroupsToRelease = collectGroupsToRelease( - Region, BlockSize, AllocatedUserEnd, CompactPtrBase); - } - if (GroupsToRelease.empty()) - return 0; + BytesInFreeList = Region->MemMapInfo.AllocatedUser - + (Region->FreeListInfo.PoppedBlocks - + Region->FreeListInfo.PushedBlocks) * + BlockSize; + if (UNLIKELY(BytesInFreeList == 0)) + return false; - // Ideally, we should use a class like `ScopedUnlock`. However, this form of - // unlocking is not supported by the thread-safety analysis. See - // https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#no-alias-analysis - // for more details. - // Put it as local class so that we can mark the ctor/dtor with proper - // annotations associated to the target lock. Note that accessing the - // function variable in local class only works in thread-safety annotations. - // TODO: Implement general `ScopedUnlock` when it's supported. - class FLLockScopedUnlock { - public: - FLLockScopedUnlock(RegionInfo *Region) RELEASE(Region->FLLock) - : R(Region) { - R->FLLock.assertHeld(); - R->FLLock.unlock(); + // ==================================================================== // + // 1. Check if we have enough free blocks and if it's worth doing a page + // release. + // ==================================================================== // + if (ReleaseType != ReleaseToOS::ForceAll && + !hasChanceToReleasePages(Region, BlockSize, BytesInFreeList, + ReleaseType)) { + return 0; } - ~FLLockScopedUnlock() ACQUIRE(Region->FLLock) { R->FLLock.lock(); } - private: - RegionInfo *R; - }; + // ==================================================================== // + // 2. Determine which groups can release the pages. Use a heuristic to + // gather groups that are candidates for doing a release. + // ==================================================================== // + if (ReleaseType == ReleaseToOS::ForceAll) { + GroupsToRelease = Region->FreeListInfo.BlockList; + Region->FreeListInfo.BlockList.clear(); + } else { + GroupsToRelease = + collectGroupsToRelease(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId)); + } + if (GroupsToRelease.empty()) + return 0; + } // Note that we have extracted the `GroupsToRelease` from region freelist. // It's safe to let pushBlocks()/popBatches() access the remaining region // freelist. In the steps 3 and 4, we will temporarily release the FLLock // and lock it again before step 5. - uptr ReleasedBytes = 0; - { - FLLockScopedUnlock UL(Region); - // ==================================================================== // - // 3. Mark the free blocks in `GroupsToRelease` in the - // `PageReleaseContext`. Then we can tell which pages are in-use by - // querying `PageReleaseContext`. - // ==================================================================== // - PageReleaseContext Context = markFreeBlocks( - Region, BlockSize, AllocatedUserEnd, CompactPtrBase, GroupsToRelease); - if (UNLIKELY(!Context.hasBlockMarked())) { - ScopedLock L(Region->FLLock); - mergeGroupsToReleaseBack(Region, GroupsToRelease); - return 0; - } - - // ==================================================================== // - // 4. Release the unused physical pages back to the OS. - // ==================================================================== // - RegionReleaseRecorder Recorder(&Region->MemMapInfo.MemMap, - Region->RegionBeg, - Context.getReleaseOffset()); - auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; - releaseFreeMemoryToOS(Context, Recorder, SkipRegion); - if (Recorder.getReleasedRangesCount() > 0) { - Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; - Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); - Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); - } - Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); - ReleasedBytes = Recorder.getReleasedBytes(); + // ==================================================================== // + // 3. Mark the free blocks in `GroupsToRelease` in the `PageReleaseContext`. + // Then we can tell which pages are in-use by querying + // `PageReleaseContext`. + // ==================================================================== // + PageReleaseContext Context = + markFreeBlocks(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId), GroupsToRelease); + if (UNLIKELY(!Context.hasBlockMarked())) { + mergeGroupsToReleaseBack(Region, GroupsToRelease); + return 0; } + // ==================================================================== // + // 4. Release the unused physical pages back to the OS. + // ==================================================================== // + RegionReleaseRecorder Recorder(&Region->MemMapInfo.MemMap, + Region->RegionBeg, + Context.getReleaseOffset()); + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + if (Recorder.getReleasedRangesCount() > 0) { + Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList; + Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); + Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); + } + Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast(); + // ====================================================================== // // 5. Merge the `GroupsToRelease` back to the freelist. // ====================================================================== // mergeGroupsToReleaseBack(Region, GroupsToRelease); - return ReleasedBytes; + return Recorder.getReleasedBytes(); } bool hasChanceToReleasePages(RegionInfo *Region, uptr BlockSize, @@ -1154,13 +1290,13 @@ private: return true; } - SinglyLinkedList + SinglyLinkedList collectGroupsToRelease(RegionInfo *Region, const uptr BlockSize, const uptr AllocatedUserEnd, const uptr CompactPtrBase) REQUIRES(Region->MMLock, Region->FLLock) { - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); const uptr PageSize = getPageSizeCached(); - SinglyLinkedList GroupsToRelease; + SinglyLinkedList GroupsToRelease; // We are examining each group and will take the minimum distance to the // release threshold as the next Region::TryReleaseThreshold(). Note that if @@ -1169,8 +1305,8 @@ private: // the comment on `SmallerBlockReleasePageDelta` for more details. uptr MinDistToThreshold = GroupSize; - for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(), - *Prev = nullptr; + for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), + *Prev = nullptr; BG != nullptr;) { // Group boundary is always GroupSize-aligned from CompactPtr base. The // layout of memory groups is like, @@ -1254,7 +1390,7 @@ private: } } - // If `BG` is the first BatchGroup in the list, we only need to advance + // If `BG` is the first BatchGroupT in the list, we only need to advance // `BG` and call FreeListInfo.BlockList::pop_front(). No update is needed // for `Prev`. // @@ -1290,7 +1426,7 @@ private: // Note that we need to advance before pushing this BatchGroup to // GroupsToRelease because it's a destructive operation. - BatchGroup *Cur = BG; + BatchGroupT *Cur = BG; BG = BG->Next; // Ideally, we may want to update this only after successful release. @@ -1323,9 +1459,9 @@ private: PageReleaseContext markFreeBlocks(RegionInfo *Region, const uptr BlockSize, const uptr AllocatedUserEnd, const uptr CompactPtrBase, - SinglyLinkedList &GroupsToRelease) + SinglyLinkedList &GroupsToRelease) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { - const uptr GroupSize = (1U << GroupSizeLog); + const uptr GroupSize = (1UL << GroupSizeLog); auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) { return decompactPtrInternal(CompactPtrBase, CompactPtr); }; @@ -1352,7 +1488,7 @@ private: if (UNLIKELY(!Context.ensurePageMapAllocated())) return Context; - for (BatchGroup &BG : GroupsToRelease) { + for (BatchGroupT &BG : GroupsToRelease) { const uptr BatchGroupBase = decompactGroupBase(CompactPtrBase, BG.CompactPtrGroupBase); const uptr BatchGroupEnd = BatchGroupBase + GroupSize; @@ -1400,8 +1536,10 @@ private: } void mergeGroupsToReleaseBack(RegionInfo *Region, - SinglyLinkedList &GroupsToRelease) - REQUIRES(Region->MMLock, Region->FLLock) { + SinglyLinkedList &GroupsToRelease) + REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { + ScopedLock L(Region->FLLock); + // After merging two freelists, we may have redundant `BatchGroup`s that // need to be recycled. The number of unused `BatchGroup`s is expected to be // small. Pick a constant which is inferred from real programs. @@ -1419,8 +1557,8 @@ private: // Merge GroupsToRelease back to the Region::FreeListInfo.BlockList. Note // that both `Region->FreeListInfo.BlockList` and `GroupsToRelease` are // sorted. - for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(), - *Prev = nullptr; + for (BatchGroupT *BG = Region->FreeListInfo.BlockList.front(), + *Prev = nullptr; ;) { if (BG == nullptr || GroupsToRelease.empty()) { if (!GroupsToRelease.empty()) @@ -1437,8 +1575,8 @@ private: continue; } - BatchGroup *Cur = GroupsToRelease.front(); - TransferBatch *UnusedTransferBatch = nullptr; + BatchGroupT *Cur = GroupsToRelease.front(); + TransferBatchT *UnusedTransferBatch = nullptr; GroupsToRelease.pop_front(); if (BG->CompactPtrGroupBase == Cur->CompactPtrGroupBase) { @@ -1454,7 +1592,7 @@ private: if (Cur->Batches.front()->getCount() == MaxCachedPerBatch) { BG->Batches.append_back(&Cur->Batches); } else { - TransferBatch *NonFullBatch = Cur->Batches.front(); + TransferBatchT *NonFullBatch = Cur->Batches.front(); Cur->Batches.pop_front(); const u16 NonFullBatchCount = NonFullBatch->getCount(); // The remaining Batches in `Cur` are full. @@ -1481,6 +1619,8 @@ private: if (UNLIKELY(Idx + NeededSlots > MaxUnusedSize)) { ScopedLock L(BatchClassRegion->FLLock); pushBatchClassBlocks(BatchClassRegion, Blocks, Idx); + if (conditionVariableEnabled()) + BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock); Idx = 0; } Blocks[Idx++] = @@ -1516,15 +1656,20 @@ private: if (Idx != 0) { ScopedLock L(BatchClassRegion->FLLock); pushBatchClassBlocks(BatchClassRegion, Blocks, Idx); + if (conditionVariableEnabled()) + BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock); } if (SCUDO_DEBUG) { - BatchGroup *Prev = Region->FreeListInfo.BlockList.front(); - for (BatchGroup *Cur = Prev->Next; Cur != nullptr; + BatchGroupT *Prev = Region->FreeListInfo.BlockList.front(); + for (BatchGroupT *Cur = Prev->Next; Cur != nullptr; Prev = Cur, Cur = Cur->Next) { CHECK_LT(Prev->CompactPtrGroupBase, Cur->CompactPtrGroupBase); } } + + if (conditionVariableEnabled()) + Region->FLLockCV.notifyAll(Region->FLLock); } // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be diff --git a/Telegram/ThirdParty/scudo/release.cpp b/Telegram/ThirdParty/scudo/release.cpp index 938bb41fa..875a2b0c1 100644 --- a/Telegram/ThirdParty/scudo/release.cpp +++ b/Telegram/ThirdParty/scudo/release.cpp @@ -10,7 +10,8 @@ namespace scudo { -BufferPool +BufferPool RegionPageMap::Buffers; } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/release.h b/Telegram/ThirdParty/scudo/release.h index 5bf963d0f..b6f76a4d2 100644 --- a/Telegram/ThirdParty/scudo/release.h +++ b/Telegram/ThirdParty/scudo/release.h @@ -80,20 +80,53 @@ private: MapPlatformData *Data = nullptr; }; -// A buffer pool which holds a fixed number of static buffers for fast buffer -// allocation. If the request size is greater than `StaticBufferSize`, it'll +class FragmentationRecorder { +public: + FragmentationRecorder() = default; + + uptr getReleasedPagesCount() const { return ReleasedPagesCount; } + + void releasePageRangeToOS(uptr From, uptr To) { + DCHECK_EQ((To - From) % getPageSizeCached(), 0U); + ReleasedPagesCount += (To - From) / getPageSizeCached(); + } + +private: + uptr ReleasedPagesCount = 0; +}; + +// A buffer pool which holds a fixed number of static buffers of `uptr` elements +// for fast buffer allocation. If the request size is greater than +// `StaticBufferNumElements` or if all the static buffers are in use, it'll // delegate the allocation to map(). -template class BufferPool { +template +class BufferPool { public: // Preserve 1 bit in the `Mask` so that we don't need to do zero-check while // extracting the least significant bit from the `Mask`. static_assert(StaticBufferCount < SCUDO_WORDSIZE, ""); - static_assert(isAligned(StaticBufferSize, SCUDO_CACHE_LINE_SIZE), ""); + static_assert(isAligned(StaticBufferNumElements * sizeof(uptr), + SCUDO_CACHE_LINE_SIZE), + ""); - // Return a buffer which is at least `BufferSize`. - uptr *getBuffer(const uptr BufferSize) { - if (UNLIKELY(BufferSize > StaticBufferSize)) - return getDynamicBuffer(BufferSize); + struct Buffer { + // Pointer to the buffer's memory, or nullptr if no buffer was allocated. + uptr *Data = nullptr; + + // The index of the underlying static buffer, or StaticBufferCount if this + // buffer was dynamically allocated. This value is initially set to a poison + // value to aid debugging. + uptr BufferIndex = ~static_cast(0); + + // Only valid if BufferIndex == StaticBufferCount. + MemMapT MemMap = {}; + }; + + // Return a zero-initialized buffer which can contain at least the given + // number of elements, or nullptr on failure. + Buffer getBuffer(const uptr NumElements) { + if (UNLIKELY(NumElements > StaticBufferNumElements)) + return getDynamicBuffer(NumElements); uptr index; { @@ -108,69 +141,55 @@ public: } if (index >= StaticBufferCount) - return getDynamicBuffer(BufferSize); + return getDynamicBuffer(NumElements); - const uptr Offset = index * StaticBufferSize; - memset(&RawBuffer[Offset], 0, StaticBufferSize); - return &RawBuffer[Offset]; + Buffer Buf; + Buf.Data = &RawBuffer[index * StaticBufferNumElements]; + Buf.BufferIndex = index; + memset(Buf.Data, 0, StaticBufferNumElements * sizeof(uptr)); + return Buf; } - void releaseBuffer(uptr *Buffer, const uptr BufferSize) { - const uptr index = getStaticBufferIndex(Buffer, BufferSize); - if (index < StaticBufferCount) { + void releaseBuffer(Buffer Buf) { + DCHECK_NE(Buf.Data, nullptr); + DCHECK_LE(Buf.BufferIndex, StaticBufferCount); + if (Buf.BufferIndex != StaticBufferCount) { ScopedLock L(Mutex); - DCHECK_EQ((Mask & (static_cast(1) << index)), 0U); - Mask |= static_cast(1) << index; + DCHECK_EQ((Mask & (static_cast(1) << Buf.BufferIndex)), 0U); + Mask |= static_cast(1) << Buf.BufferIndex; } else { - unmap(reinterpret_cast(Buffer), - roundUp(BufferSize, getPageSizeCached())); + Buf.MemMap.unmap(Buf.MemMap.getBase(), Buf.MemMap.getCapacity()); } } - bool isStaticBufferTestOnly(uptr *Buffer, uptr BufferSize) { - return getStaticBufferIndex(Buffer, BufferSize) < StaticBufferCount; + bool isStaticBufferTestOnly(const Buffer &Buf) { + DCHECK_NE(Buf.Data, nullptr); + DCHECK_LE(Buf.BufferIndex, StaticBufferCount); + return Buf.BufferIndex != StaticBufferCount; } private: - uptr getStaticBufferIndex(uptr *Buffer, uptr BufferSize) { - if (UNLIKELY(BufferSize > StaticBufferSize)) - return StaticBufferCount; - - const uptr BufferBase = reinterpret_cast(Buffer); - const uptr RawBufferBase = reinterpret_cast(RawBuffer); - - if (BufferBase < RawBufferBase || - BufferBase >= RawBufferBase + sizeof(RawBuffer)) { - return StaticBufferCount; - } - - DCHECK_LE(BufferSize, StaticBufferSize); - DCHECK_LE(BufferBase + BufferSize, RawBufferBase + sizeof(RawBuffer)); - DCHECK_EQ((BufferBase - RawBufferBase) % StaticBufferSize, 0U); - - const uptr index = - (BufferBase - RawBufferBase) / (StaticBufferSize * sizeof(uptr)); - DCHECK_LT(index, StaticBufferCount); - return index; - } - - uptr *getDynamicBuffer(const uptr BufferSize) { + Buffer getDynamicBuffer(const uptr NumElements) { // When using a heap-based buffer, precommit the pages backing the // Vmar by passing |MAP_PRECOMMIT| flag. This allows an optimization // where page fault exceptions are skipped as the allocated memory // is accessed. So far, this is only enabled on Fuchsia. It hasn't proven a // performance benefit on other platforms. const uptr MmapFlags = MAP_ALLOWNOMEM | (SCUDO_FUCHSIA ? MAP_PRECOMMIT : 0); - return reinterpret_cast( - map(nullptr, roundUp(BufferSize, getPageSizeCached()), "scudo:counters", - MmapFlags, &MapData)); + const uptr MappedSize = + roundUp(NumElements * sizeof(uptr), getPageSizeCached()); + Buffer Buf; + if (Buf.MemMap.map(/*Addr=*/0, MappedSize, "scudo:counters", MmapFlags)) { + Buf.Data = reinterpret_cast(Buf.MemMap.getBase()); + Buf.BufferIndex = StaticBufferCount; + } + return Buf; } HybridMutex Mutex; // '1' means that buffer index is not used. '0' means the buffer is in use. uptr Mask GUARDED_BY(Mutex) = ~static_cast(0); - uptr RawBuffer[StaticBufferCount * StaticBufferSize] GUARDED_BY(Mutex); - [[no_unique_address]] MapPlatformData MapData = {}; + uptr RawBuffer[StaticBufferCount * StaticBufferNumElements] GUARDED_BY(Mutex); }; // A Region page map is used to record the usage of pages in the regions. It @@ -185,23 +204,17 @@ private: class RegionPageMap { public: RegionPageMap() - : Regions(0), - NumCounters(0), - CounterSizeBitsLog(0), - CounterMask(0), - PackingRatioLog(0), - BitOffsetMask(0), - SizePerRegion(0), - BufferSize(0), - Buffer(nullptr) {} + : Regions(0), NumCounters(0), CounterSizeBitsLog(0), CounterMask(0), + PackingRatioLog(0), BitOffsetMask(0), SizePerRegion(0), + BufferNumElements(0) {} RegionPageMap(uptr NumberOfRegions, uptr CountersPerRegion, uptr MaxValue) { reset(NumberOfRegions, CountersPerRegion, MaxValue); } ~RegionPageMap() { if (!isAllocated()) return; - Buffers.releaseBuffer(Buffer, BufferSize); - Buffer = nullptr; + Buffers.releaseBuffer(Buffer); + Buffer = {}; } // Lock of `StaticBuffer` is acquired conditionally and there's no easy way to @@ -216,7 +229,7 @@ public: Regions = NumberOfRegion; NumCounters = CountersPerRegion; - constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL; + constexpr uptr MaxCounterBits = sizeof(*Buffer.Data) * 8UL; // Rounding counter storage size up to the power of two allows for using // bit shifts calculating particular counter's Index and offset. const uptr CounterSizeBits = @@ -233,11 +246,11 @@ public: SizePerRegion = roundUp(NumCounters, static_cast(1U) << PackingRatioLog) >> PackingRatioLog; - BufferSize = SizePerRegion * sizeof(*Buffer) * Regions; - Buffer = Buffers.getBuffer(BufferSize); + BufferNumElements = SizePerRegion * Regions; + Buffer = Buffers.getBuffer(BufferNumElements); } - bool isAllocated() const { return !!Buffer; } + bool isAllocated() const { return Buffer.Data != nullptr; } uptr getCount() const { return NumCounters; } @@ -246,7 +259,8 @@ public: DCHECK_LT(I, NumCounters); const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; - return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask; + return (Buffer.Data[Region * SizePerRegion + Index] >> BitOffset) & + CounterMask; } void inc(uptr Region, uptr I) const { @@ -255,8 +269,8 @@ public: const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); DCHECK_EQ(isAllCounted(Region, I), false); - Buffer[Region * SizePerRegion + Index] += static_cast(1U) - << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] += static_cast(1U) + << BitOffset; } void incN(uptr Region, uptr I, uptr N) const { @@ -267,7 +281,7 @@ public: const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); DCHECK_EQ(isAllCounted(Region, I), false); - Buffer[Region * SizePerRegion + Index] += N << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] += N << BitOffset; } void incRange(uptr Region, uptr From, uptr To) const { @@ -286,7 +300,7 @@ public: const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); - Buffer[Region * SizePerRegion + Index] |= CounterMask << BitOffset; + Buffer.Data[Region * SizePerRegion + Index] |= CounterMask << BitOffset; } void setAsAllCountedRange(uptr Region, uptr From, uptr To) const { DCHECK_LE(From, To); @@ -309,9 +323,16 @@ public: return get(Region, I) == CounterMask; } - uptr getBufferSize() const { return BufferSize; } + uptr getBufferNumElements() const { return BufferNumElements; } private: + // We may consider making this configurable if there are cases which may + // benefit from this. + static const uptr StaticBufferCount = 2U; + static const uptr StaticBufferNumElements = 512U; + using BufferPoolT = BufferPool; + static BufferPoolT Buffers; + uptr Regions; uptr NumCounters; uptr CounterSizeBitsLog; @@ -320,14 +341,8 @@ private: uptr BitOffsetMask; uptr SizePerRegion; - uptr BufferSize; - uptr *Buffer; - - // We may consider making this configurable if there are cases which may - // benefit from this. - static const uptr StaticBufferCount = 2U; - static const uptr StaticBufferSize = 512U; - static BufferPool Buffers; + uptr BufferNumElements; + BufferPoolT::Buffer Buffer; }; template class FreePagesRangeTracker { diff --git a/Telegram/ThirdParty/scudo/report.cpp b/Telegram/ThirdParty/scudo/report.cpp index 81b3dce4e..9cef0adc0 100644 --- a/Telegram/ThirdParty/scudo/report.cpp +++ b/Telegram/ThirdParty/scudo/report.cpp @@ -24,11 +24,7 @@ public: Message.vappend(Format, Args); va_end(Args); } - NORETURN ~ScopedErrorReport() { - outputRaw(Message.data()); - setAbortMessage(Message.data()); - die(); - } + NORETURN ~ScopedErrorReport() { reportRawError(Message.data()); } private: ScopedString Message; @@ -36,18 +32,6 @@ private: inline void NORETURN trap() { __builtin_trap(); } -void NORETURN reportSoftRSSLimit(uptr RssLimitMb) { - ScopedErrorReport Report; - Report.append("Soft RSS limit of %zu MB exhausted, current RSS is %zu MB\n", - RssLimitMb, GetRSS() >> 20); -} - -void NORETURN reportHardRSSLimit(uptr RssLimitMb) { - ScopedErrorReport Report; - Report.append("Hard RSS limit of %zu MB exhausted, current RSS is %zu MB\n", - RssLimitMb, GetRSS() >> 20); -} - // This could potentially be called recursively if a CHECK fails in the reports. void NORETURN reportCheckFailed(const char *File, int Line, const char *Condition, u64 Value1, u64 Value2) { @@ -67,6 +51,13 @@ void NORETURN reportError(const char *Message) { Report.append("%s\n", Message); } +// Generic fatal error message without ScopedString. +void NORETURN reportRawError(const char *Message) { + outputRaw(Message); + setAbortMessage(Message); + die(); +} + void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) { ScopedErrorReport Report; Report.append("invalid value for %s option: '%s'\n", FlagType, Value); @@ -79,14 +70,6 @@ void NORETURN reportHeaderCorruption(void *Ptr) { Report.append("corrupted chunk header at address %p\n", Ptr); } -// Two threads have attempted to modify a chunk header at the same time. This is -// symptomatic of a race-condition in the application code, or general lack of -// proper locking. -void NORETURN reportHeaderRace(void *Ptr) { - ScopedErrorReport Report; - Report.append("race on chunk header at address %p\n", Ptr); -} - // The allocator was compiled with parameters that conflict with field size // requirements. void NORETURN reportSanityCheckError(const char *Field) { diff --git a/Telegram/ThirdParty/scudo/report.h b/Telegram/ThirdParty/scudo/report.h index 3a78ab64b..a510fdaeb 100644 --- a/Telegram/ThirdParty/scudo/report.h +++ b/Telegram/ThirdParty/scudo/report.h @@ -15,15 +15,17 @@ namespace scudo { // Reports are *fatal* unless stated otherwise. -// Generic error. +// Generic error, adds newline to end of message. void NORETURN reportError(const char *Message); +// Generic error, but the message is not modified. +void NORETURN reportRawError(const char *Message); + // Flags related errors. void NORETURN reportInvalidFlag(const char *FlagType, const char *Value); // Chunk header related errors. void NORETURN reportHeaderCorruption(void *Ptr); -void NORETURN reportHeaderRace(void *Ptr); // Sanity checks related error. void NORETURN reportSanityCheckError(const char *Field); @@ -34,8 +36,6 @@ void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, uptr MaxSize); void NORETURN reportOutOfBatchClass(); void NORETURN reportOutOfMemory(uptr RequestedSize); -void NORETURN reportSoftRSSLimit(uptr RssLimitMb); -void NORETURN reportHardRSSLimit(uptr RssLimitMb); enum class AllocatorAction : u8 { Recycling, Deallocating, diff --git a/Telegram/ThirdParty/scudo/report_linux.cpp b/Telegram/ThirdParty/scudo/report_linux.cpp new file mode 100644 index 000000000..6a983036e --- /dev/null +++ b/Telegram/ThirdParty/scudo/report_linux.cpp @@ -0,0 +1,58 @@ +//===-- report_linux.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_LINUX || SCUDO_TRUSTY + +#include "common.h" +#include "internal_defs.h" +#include "report.h" +#include "report_linux.h" +#include "string_utils.h" + +#include +#include +#include + +namespace scudo { + +// Fatal internal map() error (potentially OOM related). +void NORETURN reportMapError(uptr SizeIfOOM) { + char Error[128] = "Scudo ERROR: internal map failure\n"; + if (SizeIfOOM) { + formatString( + Error, sizeof(Error), + "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n", + SizeIfOOM >> 10); + } + reportRawError(Error); +} + +void NORETURN reportUnmapError(uptr Addr, uptr Size) { + char Error[128]; + formatString(Error, sizeof(Error), + "Scudo ERROR: internal unmap failure (error desc=%s) Addr 0x%zx " + "Size %zu\n", + strerror(errno), Addr, Size); + reportRawError(Error); +} + +void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot) { + char Error[128]; + formatString( + Error, sizeof(Error), + "Scudo ERROR: internal protect failure (error desc=%s) Addr 0x%zx " + "Size %zu Prot %x\n", + strerror(errno), Addr, Size, Prot); + reportRawError(Error); +} + +} // namespace scudo + +#endif // SCUDO_LINUX || SCUDO_TRUSTY diff --git a/Telegram/ThirdParty/scudo/report_linux.h b/Telegram/ThirdParty/scudo/report_linux.h new file mode 100644 index 000000000..aa0bb247e --- /dev/null +++ b/Telegram/ThirdParty/scudo/report_linux.h @@ -0,0 +1,34 @@ +//===-- report_linux.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_REPORT_LINUX_H_ +#define SCUDO_REPORT_LINUX_H_ + +#include "platform.h" + +#if SCUDO_LINUX || SCUDO_TRUSTY + +#include "internal_defs.h" + +namespace scudo { + +// Report a fatal error when a map call fails. SizeIfOOM shall +// hold the requested size on an out-of-memory error, 0 otherwise. +void NORETURN reportMapError(uptr SizeIfOOM = 0); + +// Report a fatal error when an unmap call fails. +void NORETURN reportUnmapError(uptr Addr, uptr Size); + +// Report a fatal error when a mprotect call fails. +void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot); + +} // namespace scudo + +#endif // SCUDO_LINUX || SCUDO_TRUSTY + +#endif // SCUDO_REPORT_LINUX_H_ diff --git a/Telegram/ThirdParty/scudo/rss_limit_checker.cpp b/Telegram/ThirdParty/scudo/rss_limit_checker.cpp deleted file mode 100644 index f428386b7..000000000 --- a/Telegram/ThirdParty/scudo/rss_limit_checker.cpp +++ /dev/null @@ -1,37 +0,0 @@ -//===-- common.cpp ----------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "rss_limit_checker.h" -#include "atomic_helpers.h" -#include "string_utils.h" - -namespace scudo { - -void RssLimitChecker::check(u64 NextCheck) { - // The interval for the checks is 250ms. - static constexpr u64 CheckInterval = 250 * 1000000; - - // Early return in case another thread already did the calculation. - if (!atomic_compare_exchange_strong(&RssNextCheckAtNS, &NextCheck, - getMonotonicTime() + CheckInterval, - memory_order_relaxed)) { - return; - } - - const uptr CurrentRssMb = GetRSS() >> 20; - - RssLimitExceeded Result = RssLimitExceeded::Neither; - if (UNLIKELY(HardRssLimitMb && HardRssLimitMb < CurrentRssMb)) - Result = RssLimitExceeded::Hard; - else if (UNLIKELY(SoftRssLimitMb && SoftRssLimitMb < CurrentRssMb)) - Result = RssLimitExceeded::Soft; - - atomic_store_relaxed(&RssLimitStatus, static_cast(Result)); -} - -} // namespace scudo diff --git a/Telegram/ThirdParty/scudo/rss_limit_checker.h b/Telegram/ThirdParty/scudo/rss_limit_checker.h deleted file mode 100644 index 29dc063f3..000000000 --- a/Telegram/ThirdParty/scudo/rss_limit_checker.h +++ /dev/null @@ -1,63 +0,0 @@ -//===-- common.h ------------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_RSS_LIMIT_CHECKER_H_ -#define SCUDO_RSS_LIMIT_CHECKER_H_ - -#include "atomic_helpers.h" -#include "common.h" -#include "internal_defs.h" - -namespace scudo { - -class RssLimitChecker { -public: - enum RssLimitExceeded { - Neither, - Soft, - Hard, - }; - - void init(int SoftRssLimitMb, int HardRssLimitMb) { - CHECK_GE(SoftRssLimitMb, 0); - CHECK_GE(HardRssLimitMb, 0); - this->SoftRssLimitMb = static_cast(SoftRssLimitMb); - this->HardRssLimitMb = static_cast(HardRssLimitMb); - } - - // Opportunistic RSS limit check. This will update the RSS limit status, if - // it can, every 250ms, otherwise it will just return the current one. - RssLimitExceeded getRssLimitExceeded() { - if (!HardRssLimitMb && !SoftRssLimitMb) - return RssLimitExceeded::Neither; - - u64 NextCheck = atomic_load_relaxed(&RssNextCheckAtNS); - u64 Now = getMonotonicTime(); - - if (UNLIKELY(Now >= NextCheck)) - check(NextCheck); - - return static_cast(atomic_load_relaxed(&RssLimitStatus)); - } - - uptr getSoftRssLimit() const { return SoftRssLimitMb; } - uptr getHardRssLimit() const { return HardRssLimitMb; } - -private: - void check(u64 NextCheck); - - uptr SoftRssLimitMb = 0; - uptr HardRssLimitMb = 0; - - atomic_u64 RssNextCheckAtNS = {}; - atomic_u8 RssLimitStatus = {}; -}; - -} // namespace scudo - -#endif // SCUDO_RSS_LIMIT_CHECKER_H_ diff --git a/Telegram/ThirdParty/scudo/secondary.h b/Telegram/ThirdParty/scudo/secondary.h index 105b154b5..f52a4188b 100644 --- a/Telegram/ThirdParty/scudo/secondary.h +++ b/Telegram/ThirdParty/scudo/secondary.h @@ -72,11 +72,26 @@ static inline void unmap(LargeBlock::Header *H) { MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); } +namespace { +struct CachedBlock { + uptr CommitBase = 0; + uptr CommitSize = 0; + uptr BlockBegin = 0; + MemMapT MemMap = {}; + u64 Time = 0; + + bool isValid() { return CommitBase != 0; } + + void invalidate() { CommitBase = 0; } +}; +} // namespace + template class MapAllocatorNoCache { public: void init(UNUSED s32 ReleaseToOsInterval) {} bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, - UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) { + UNUSED uptr HeadersSize, UNUSED LargeBlock::Header **H, + UNUSED bool *Zeroed) { return false; } void store(UNUSED Options Options, LargeBlock::Header *H) { unmap(H); } @@ -102,20 +117,22 @@ public: static const uptr MaxUnusedCachePages = 4U; template -void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize, +bool mapSecondary(const Options &Options, uptr CommitBase, uptr CommitSize, uptr AllocPos, uptr Flags, MemMapT &MemMap) { + Flags |= MAP_RESIZABLE; + Flags |= MAP_ALLOWNOMEM; + const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached(); if (useMemoryTagging(Options) && CommitSize > MaxUnusedCacheBytes) { const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes); - MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary", - MAP_RESIZABLE | MAP_MEMTAG | Flags); - MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos, - "scudo:secondary", MAP_RESIZABLE | Flags); + return MemMap.remap(CommitBase, UntaggedPos - CommitBase, "scudo:secondary", + MAP_MEMTAG | Flags) && + MemMap.remap(UntaggedPos, CommitBase + CommitSize - UntaggedPos, + "scudo:secondary", Flags); } else { const uptr RemapFlags = - MAP_RESIZABLE | (useMemoryTagging(Options) ? MAP_MEMTAG : 0) | - Flags; - MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags); + (useMemoryTagging(Options) ? MAP_MEMTAG : 0) | Flags; + return MemMap.remap(CommitBase, CommitSize, "scudo:secondary", RemapFlags); } } @@ -138,17 +155,24 @@ public: void getStats(ScopedString *Str) { ScopedLock L(Mutex); + uptr Integral; + uptr Fractional; + computePercentage(SuccessfulRetrieves, CallsToRetrieve, &Integral, + &Fractional); Str->append("Stats: MapAllocatorCache: EntriesCount: %d, " "MaxEntriesCount: %u, MaxEntrySize: %zu\n", EntriesCount, atomic_load_relaxed(&MaxEntriesCount), atomic_load_relaxed(&MaxEntrySize)); + Str->append("Stats: CacheRetrievalStats: SuccessRate: %u/%u " + "(%zu.%02zu%%)\n", + SuccessfulRetrieves, CallsToRetrieve, Integral, Fractional); for (CachedBlock Entry : Entries) { - if (!Entry.CommitBase) + if (!Entry.isValid()) continue; Str->append("StartBlockAddress: 0x%zx, EndBlockAddress: 0x%zx, " - "BlockSize: %zu\n", + "BlockSize: %zu %s\n", Entry.CommitBase, Entry.CommitBase + Entry.CommitSize, - Entry.CommitSize); + Entry.CommitSize, Entry.Time == 0 ? "[R]" : ""); } } @@ -166,7 +190,7 @@ public: setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } - void store(Options Options, LargeBlock::Header *H) EXCLUDES(Mutex) { + void store(const Options &Options, LargeBlock::Header *H) EXCLUDES(Mutex) { if (!canCache(H->CommitSize)) return unmap(H); @@ -195,7 +219,7 @@ public: MAP_NOACCESS); } } else if (Interval == 0) { - Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); + Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } do { @@ -210,7 +234,7 @@ public: if (CacheConfig::QuarantineSize && useMemoryTagging(Options)) { QuarantinePos = (QuarantinePos + 1) % Max(CacheConfig::QuarantineSize, 1u); - if (!Quarantine[QuarantinePos].CommitBase) { + if (!Quarantine[QuarantinePos].isValid()) { Quarantine[QuarantinePos] = Entry; return; } @@ -225,7 +249,7 @@ public: EmptyCache = true; } else { for (u32 I = 0; I < MaxCount; I++) { - if (Entries[I].CommitBase) + if (Entries[I].isValid()) continue; if (I != 0) Entries[I] = Entries[0]; @@ -246,26 +270,31 @@ public: Entry.MemMap.unmap(Entry.MemMap.getBase(), Entry.MemMap.getCapacity()); } - bool retrieve(Options Options, uptr Size, uptr Alignment, + bool retrieve(Options Options, uptr Size, uptr Alignment, uptr HeadersSize, LargeBlock::Header **H, bool *Zeroed) EXCLUDES(Mutex) { const uptr PageSize = getPageSizeCached(); const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); + // 10% of the requested size proved to be the optimal choice for + // retrieving cached blocks after testing several options. + constexpr u32 FragmentedBytesDivisor = 10; bool Found = false; CachedBlock Entry; - uptr HeaderPos = 0; + uptr EntryHeaderPos = 0; { ScopedLock L(Mutex); + CallsToRetrieve++; if (EntriesCount == 0) return false; + u32 OptimalFitIndex = 0; + uptr MinDiff = UINTPTR_MAX; for (u32 I = 0; I < MaxCount; I++) { - const uptr CommitBase = Entries[I].CommitBase; - if (!CommitBase) + if (!Entries[I].isValid()) continue; + const uptr CommitBase = Entries[I].CommitBase; const uptr CommitSize = Entries[I].CommitSize; const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment); - HeaderPos = - AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + const uptr HeaderPos = AllocPos - HeadersSize; if (HeaderPos > CommitBase + CommitSize) continue; if (HeaderPos < CommitBase || @@ -273,17 +302,36 @@ public: continue; } Found = true; - Entry = Entries[I]; - Entries[I].CommitBase = 0; + const uptr Diff = HeaderPos - CommitBase; + // immediately use a cached block if it's size is close enough to the + // requested size. + const uptr MaxAllowedFragmentedBytes = + (CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor; + if (Diff <= MaxAllowedFragmentedBytes) { + OptimalFitIndex = I; + EntryHeaderPos = HeaderPos; + break; + } + // keep track of the smallest cached block + // that is greater than (AllocSize + HeaderSize) + if (Diff > MinDiff) + continue; + OptimalFitIndex = I; + MinDiff = Diff; + EntryHeaderPos = HeaderPos; + } + if (Found) { + Entry = Entries[OptimalFitIndex]; + Entries[OptimalFitIndex].invalidate(); EntriesCount--; - break; + SuccessfulRetrieves++; } } if (!Found) return false; *H = reinterpret_cast( - LargeBlock::addHeaderTag(HeaderPos)); + LargeBlock::addHeaderTag(EntryHeaderPos)); *Zeroed = Entry.Time == 0; if (useMemoryTagging(Options)) Entry.MemMap.setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0); @@ -295,8 +343,7 @@ public: } else if (Entry.BlockBegin < NewBlockBegin) { storeTags(Entry.BlockBegin, NewBlockBegin); } else { - storeTags(untagPointer(NewBlockBegin), - untagPointer(Entry.BlockBegin)); + storeTags(untagPointer(NewBlockBegin), untagPointer(Entry.BlockBegin)); } } (*H)->CommitBase = Entry.CommitBase; @@ -338,15 +385,15 @@ public: void disableMemoryTagging() EXCLUDES(Mutex) { ScopedLock L(Mutex); for (u32 I = 0; I != CacheConfig::QuarantineSize; ++I) { - if (Quarantine[I].CommitBase) { + if (Quarantine[I].isValid()) { MemMapT &MemMap = Quarantine[I].MemMap; MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); - Quarantine[I].CommitBase = 0; + Quarantine[I].invalidate(); } } const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount); for (u32 I = 0; I < MaxCount; I++) { - if (Entries[I].CommitBase) { + if (Entries[I].isValid()) { Entries[I].MemMap.setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0); } @@ -367,10 +414,10 @@ private: { ScopedLock L(Mutex); for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) { - if (!Entries[I].CommitBase) + if (!Entries[I].isValid()) continue; MapInfo[N] = Entries[I].MemMap; - Entries[I].CommitBase = 0; + Entries[I].invalidate(); N++; } EntriesCount = 0; @@ -382,23 +429,15 @@ private: } } - struct CachedBlock { - uptr CommitBase = 0; - uptr CommitSize = 0; - uptr BlockBegin = 0; - MemMapT MemMap = {}; - u64 Time = 0; - }; - void releaseIfOlderThan(CachedBlock &Entry, u64 Time) REQUIRES(Mutex) { - if (!Entry.CommitBase || !Entry.Time) + if (!Entry.isValid() || !Entry.Time) return; if (Entry.Time > Time) { if (OldestTime == 0 || Entry.Time < OldestTime) OldestTime = Entry.Time; return; } - Entry.MemMap.releasePagesToOS(Entry.CommitBase, Entry.CommitSize); + Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, Entry.CommitSize); Entry.Time = 0; } @@ -421,6 +460,8 @@ private: u64 OldestTime GUARDED_BY(Mutex) = 0; u32 IsFullEvents GUARDED_BY(Mutex) = 0; atomic_s32 ReleaseToOsIntervalMs = {}; + u32 CallsToRetrieve GUARDED_BY(Mutex) = 0; + u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0; CachedBlock Entries[CacheConfig::EntriesArraySize] GUARDED_BY(Mutex) = {}; NonZeroLengthArray @@ -439,11 +480,11 @@ public: S->link(&Stats); } - void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0, + void *allocate(const Options &Options, uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, FillContentsMode FillContents = NoFill); - void deallocate(Options Options, void *Ptr); + void deallocate(const Options &Options, void *Ptr); static uptr getBlockEnd(void *Ptr) { auto *B = LargeBlock::getHeader(Ptr); @@ -454,6 +495,10 @@ public: return getBlockEnd(Ptr) - reinterpret_cast(Ptr); } + static constexpr uptr getHeadersSize() { + return Chunk::getHeaderSize() + LargeBlock::getHeaderSize(); + } + void disable() NO_THREAD_SAFETY_ANALYSIS { Mutex.lock(); Cache.disable(); @@ -494,6 +539,7 @@ private: DoublyLinkedList InUseBlocks GUARDED_BY(Mutex); uptr AllocatedBytes GUARDED_BY(Mutex) = 0; uptr FreedBytes GUARDED_BY(Mutex) = 0; + uptr FragmentedBytes GUARDED_BY(Mutex) = 0; uptr LargestSize GUARDED_BY(Mutex) = 0; u32 NumberOfAllocs GUARDED_BY(Mutex) = 0; u32 NumberOfFrees GUARDED_BY(Mutex) = 0; @@ -512,24 +558,23 @@ private: // the committed memory will amount to something close to Size - AlignmentHint // (pending rounding and headers). template -void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, - uptr *BlockEndPtr, +void *MapAllocator::allocate(const Options &Options, uptr Size, + uptr Alignment, uptr *BlockEndPtr, FillContentsMode FillContents) { if (Options.get(OptionBit::AddLargeAllocationSlack)) Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG; Alignment = Max(Alignment, uptr(1U) << SCUDO_MIN_ALIGNMENT_LOG); const uptr PageSize = getPageSizeCached(); - uptr RoundedSize = - roundUp(roundUp(Size, Alignment) + LargeBlock::getHeaderSize() + - Chunk::getHeaderSize(), - PageSize); - if (Alignment > PageSize) - RoundedSize += Alignment - PageSize; - if (Alignment < PageSize && Cache.canCache(RoundedSize)) { + // Note that cached blocks may have aligned address already. Thus we simply + // pass the required size (`Size` + `getHeadersSize()`) to do cache look up. + const uptr MinNeededSizeForCache = roundUp(Size + getHeadersSize(), PageSize); + + if (Alignment < PageSize && Cache.canCache(MinNeededSizeForCache)) { LargeBlock::Header *H; bool Zeroed; - if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) { + if (Cache.retrieve(Options, Size, Alignment, getHeadersSize(), &H, + &Zeroed)) { const uptr BlockEnd = H->CommitBase + H->CommitSize; if (BlockEndPtr) *BlockEndPtr = BlockEnd; @@ -545,6 +590,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, ScopedLock L(Mutex); InUseBlocks.push_back(H); AllocatedBytes += H->CommitSize; + FragmentedBytes += H->MemMap.getCapacity() - H->CommitSize; NumberOfAllocs++; Stats.add(StatAllocated, H->CommitSize); Stats.add(StatMapped, H->MemMap.getCapacity()); @@ -553,16 +599,22 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, } } + uptr RoundedSize = + roundUp(roundUp(Size, Alignment) + getHeadersSize(), PageSize); + if (Alignment > PageSize) + RoundedSize += Alignment - PageSize; + ReservedMemoryT ReservedMemory; const uptr MapSize = RoundedSize + 2 * PageSize; - ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, MAP_ALLOWNOMEM); + if (UNLIKELY(!ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr, + MAP_ALLOWNOMEM))) { + return nullptr; + } // Take the entire ownership of reserved region. MemMapT MemMap = ReservedMemory.dispatch(ReservedMemory.getBase(), ReservedMemory.getCapacity()); uptr MapBase = MemMap.getBase(); - if (UNLIKELY(!MapBase)) - return nullptr; uptr CommitBase = MapBase + PageSize; uptr MapEnd = MapBase + MapSize; @@ -592,9 +644,12 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, const uptr CommitSize = MapEnd - PageSize - CommitBase; const uptr AllocPos = roundDown(CommitBase + CommitSize - Size, Alignment); - mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, MemMap); - const uptr HeaderPos = - AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize(); + if (!mapSecondary(Options, CommitBase, CommitSize, AllocPos, 0, + MemMap)) { + MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + return nullptr; + } + const uptr HeaderPos = AllocPos - getHeadersSize(); LargeBlock::Header *H = reinterpret_cast( LargeBlock::addHeaderTag(HeaderPos)); if (useMemoryTagging(Options)) @@ -609,6 +664,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, ScopedLock L(Mutex); InUseBlocks.push_back(H); AllocatedBytes += CommitSize; + FragmentedBytes += H->MemMap.getCapacity() - CommitSize; if (LargestSize < CommitSize) LargestSize = CommitSize; NumberOfAllocs++; @@ -619,7 +675,7 @@ void *MapAllocator::allocate(Options Options, uptr Size, uptr Alignment, } template -void MapAllocator::deallocate(Options Options, void *Ptr) +void MapAllocator::deallocate(const Options &Options, void *Ptr) EXCLUDES(Mutex) { LargeBlock::Header *H = LargeBlock::getHeader(Ptr); const uptr CommitSize = H->CommitSize; @@ -627,6 +683,7 @@ void MapAllocator::deallocate(Options Options, void *Ptr) ScopedLock L(Mutex); InUseBlocks.remove(H); FreedBytes += CommitSize; + FragmentedBytes -= H->MemMap.getCapacity() - CommitSize; NumberOfFrees++; Stats.sub(StatAllocated, CommitSize); Stats.sub(StatMapped, H->MemMap.getCapacity()); @@ -638,10 +695,11 @@ template void MapAllocator::getStats(ScopedString *Str) EXCLUDES(Mutex) { ScopedLock L(Mutex); Str->append("Stats: MapAllocator: allocated %u times (%zuK), freed %u times " - "(%zuK), remains %u (%zuK) max %zuM\n", + "(%zuK), remains %u (%zuK) max %zuM, Fragmented %zuK\n", NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, NumberOfAllocs - NumberOfFrees, - (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20); + (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20, + FragmentedBytes >> 10); Cache.getStats(Str); } diff --git a/Telegram/ThirdParty/scudo/size_class_map.h b/Telegram/ThirdParty/scudo/size_class_map.h index 2a6e298f9..4138885de 100644 --- a/Telegram/ThirdParty/scudo/size_class_map.h +++ b/Telegram/ThirdParty/scudo/size_class_map.h @@ -254,7 +254,7 @@ struct AndroidSizeClassConfig { static const u16 MaxNumCachedHint = 13; static const uptr MaxBytesCachedLog = 13; - static constexpr u32 Classes[] = { + static constexpr uptr Classes[] = { 0x00020, 0x00030, 0x00040, 0x00050, 0x00060, 0x00070, 0x00090, 0x000b0, 0x000c0, 0x000e0, 0x00120, 0x00160, 0x001c0, 0x00250, 0x00320, 0x00450, 0x00670, 0x00830, 0x00a10, 0x00c30, 0x01010, 0x01210, 0x01bd0, 0x02210, @@ -269,7 +269,7 @@ struct AndroidSizeClassConfig { static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 13; - static constexpr u32 Classes[] = { + static constexpr uptr Classes[] = { 0x00020, 0x00030, 0x00040, 0x00050, 0x00060, 0x00070, 0x00080, 0x00090, 0x000a0, 0x000b0, 0x000c0, 0x000e0, 0x000f0, 0x00110, 0x00120, 0x00130, 0x00150, 0x00160, 0x00170, 0x00190, 0x001d0, 0x00210, 0x00240, 0x002a0, @@ -289,28 +289,6 @@ typedef TableSizeClassMap AndroidSizeClassMap; static_assert(AndroidSizeClassMap::usesCompressedLSBFormat(), ""); #endif -struct SvelteSizeClassConfig { -#if SCUDO_WORDSIZE == 64U - static const uptr NumBits = 4; - static const uptr MinSizeLog = 4; - static const uptr MidSizeLog = 8; - static const uptr MaxSizeLog = 14; - static const u16 MaxNumCachedHint = 13; - static const uptr MaxBytesCachedLog = 10; - static const uptr SizeDelta = Chunk::getHeaderSize(); -#else - static const uptr NumBits = 4; - static const uptr MinSizeLog = 3; - static const uptr MidSizeLog = 7; - static const uptr MaxSizeLog = 14; - static const u16 MaxNumCachedHint = 14; - static const uptr MaxBytesCachedLog = 10; - static const uptr SizeDelta = Chunk::getHeaderSize(); -#endif -}; - -typedef FixedSizeClassMap SvelteSizeClassMap; - struct TrustySizeClassConfig { static const uptr NumBits = 1; static const uptr MinSizeLog = 5; diff --git a/Telegram/ThirdParty/scudo/stack_depot.h b/Telegram/ThirdParty/scudo/stack_depot.h index 458198fcb..12c35eb2a 100644 --- a/Telegram/ThirdParty/scudo/stack_depot.h +++ b/Telegram/ThirdParty/scudo/stack_depot.h @@ -62,8 +62,7 @@ class StackDepot { // This is achieved by re-checking the hash of the stack trace before // returning the trace. -#ifdef SCUDO_FUZZ - // Use smaller table sizes for fuzzing in order to reduce input size. +#if SCUDO_SMALL_STACK_DEPOT static const uptr TabBits = 4; #else static const uptr TabBits = 16; @@ -72,7 +71,7 @@ class StackDepot { static const uptr TabMask = TabSize - 1; atomic_u32 Tab[TabSize] = {}; -#ifdef SCUDO_FUZZ +#if SCUDO_SMALL_STACK_DEPOT static const uptr RingBits = 4; #else static const uptr RingBits = 19; diff --git a/Telegram/ThirdParty/scudo/tests/CMakeLists.txt b/Telegram/ThirdParty/scudo/tests/CMakeLists.txt index cbb85ce3e..c6b6a1cb5 100644 --- a/Telegram/ThirdParty/scudo/tests/CMakeLists.txt +++ b/Telegram/ThirdParty/scudo/tests/CMakeLists.txt @@ -15,11 +15,15 @@ set(SCUDO_UNITTEST_CFLAGS -DGTEST_HAS_RTTI=0 -g # Extra flags for the C++ tests + -Wconversion # TODO(kostyak): find a way to make -fsized-deallocation work -Wno-mismatched-new-delete) if(COMPILER_RT_DEBUG) - list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_DEBUG=1) + list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_DEBUG=1 -DSCUDO_ENABLE_HOOKS=1) + if (NOT FUCHSIA) + list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_ENABLE_HOOKS_TESTS=1) + endif() endif() if(ANDROID) @@ -92,6 +96,7 @@ set(SCUDO_UNIT_TEST_SOURCES chunk_test.cpp combined_test.cpp common_test.cpp + condition_variable_test.cpp flags_test.cpp list_test.cpp map_test.cpp @@ -137,11 +142,3 @@ set(SCUDO_CXX_UNIT_TEST_SOURCES add_scudo_unittest(ScudoCxxUnitTest SOURCES ${SCUDO_CXX_UNIT_TEST_SOURCES} ADDITIONAL_RTOBJECTS RTScudoStandaloneCWrappers RTScudoStandaloneCxxWrappers) - -set(SCUDO_HOOKS_UNIT_TEST_SOURCES - scudo_hooks_test.cpp - scudo_unit_test_main.cpp - ) - -add_scudo_unittest(ScudoHooksUnitTest - SOURCES ${SCUDO_HOOKS_UNIT_TEST_SOURCES}) diff --git a/Telegram/ThirdParty/scudo/tests/chunk_test.cpp b/Telegram/ThirdParty/scudo/tests/chunk_test.cpp index 7a29f3c11..1b2c1eb5a 100644 --- a/Telegram/ThirdParty/scudo/tests/chunk_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/chunk_test.cpp @@ -37,29 +37,6 @@ TEST(ScudoChunkDeathTest, ChunkBasic) { free(Block); } -TEST(ScudoChunkTest, ChunkCmpXchg) { - initChecksum(); - const scudo::uptr Size = 0x100U; - scudo::Chunk::UnpackedHeader OldHeader = {}; - OldHeader.OriginOrWasZeroed = scudo::Chunk::Origin::Malloc; - OldHeader.ClassId = 0x42U; - OldHeader.SizeOrUnusedBytes = Size; - OldHeader.State = scudo::Chunk::State::Allocated; - void *Block = malloc(HeaderSize + Size); - void *P = reinterpret_cast(reinterpret_cast(Block) + - HeaderSize); - scudo::Chunk::storeHeader(Cookie, P, &OldHeader); - memset(P, 'A', Size); - scudo::Chunk::UnpackedHeader NewHeader = OldHeader; - NewHeader.State = scudo::Chunk::State::Quarantined; - scudo::Chunk::compareExchangeHeader(Cookie, P, &NewHeader, &OldHeader); - NewHeader = {}; - EXPECT_TRUE(scudo::Chunk::isValid(Cookie, P, &NewHeader)); - EXPECT_EQ(NewHeader.State, scudo::Chunk::State::Quarantined); - EXPECT_FALSE(scudo::Chunk::isValid(InvalidCookie, P, &NewHeader)); - free(Block); -} - TEST(ScudoChunkDeathTest, CorruptHeader) { initChecksum(); const scudo::uptr Size = 0x100U; diff --git a/Telegram/ThirdParty/scudo/tests/combined_test.cpp b/Telegram/ThirdParty/scudo/tests/combined_test.cpp index 79d327849..3dbd93cac 100644 --- a/Telegram/ThirdParty/scudo/tests/combined_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/combined_test.cpp @@ -12,8 +12,11 @@ #include "allocator_config.h" #include "chunk.h" #include "combined.h" +#include "condition_variable.h" #include "mem_map.h" +#include "size_class_map.h" +#include #include #include #include @@ -54,7 +57,7 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size, EXPECT_DEATH( { disableDebuggerdMaybe(); - reinterpret_cast(P)[-1] = 0xaa; + reinterpret_cast(P)[-1] = 'A'; }, ""); if (isPrimaryAllocation(Size, Alignment) @@ -63,7 +66,7 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size, EXPECT_DEATH( { disableDebuggerdMaybe(); - reinterpret_cast(P)[Size] = 0xaa; + reinterpret_cast(P)[Size] = 'A'; }, ""); } @@ -78,14 +81,70 @@ template struct TestAllocator : scudo::Allocator { } ~TestAllocator() { this->unmapTestOnly(); } - void *operator new(size_t size) { - void *p = nullptr; - EXPECT_EQ(0, posix_memalign(&p, alignof(TestAllocator), size)); - return p; + void *operator new(size_t size); + void operator delete(void *ptr); +}; + +constexpr size_t kMaxAlign = std::max({ + alignof(scudo::Allocator), +#if SCUDO_CAN_USE_PRIMARY64 + alignof(scudo::Allocator), +#endif + alignof(scudo::Allocator) +}); + +#if SCUDO_RISCV64 +// The allocator is over 4MB large. Rather than creating an instance of this on +// the heap, keep it in a global storage to reduce fragmentation from having to +// mmap this at the start of every test. +struct TestAllocatorStorage { + static constexpr size_t kMaxSize = std::max({ + sizeof(scudo::Allocator), +#if SCUDO_CAN_USE_PRIMARY64 + sizeof(scudo::Allocator), +#endif + sizeof(scudo::Allocator) + }); + + // To alleviate some problem, let's skip the thread safety analysis here. + static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS { + CHECK(size <= kMaxSize && + "Allocation size doesn't fit in the allocator storage"); + M.lock(); + return AllocatorStorage; } - void operator delete(void *ptr) { free(ptr); } + static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS { + M.assertHeld(); + M.unlock(); + ASSERT_EQ(ptr, AllocatorStorage); + } + + static scudo::HybridMutex M; + static uint8_t AllocatorStorage[kMaxSize]; }; +scudo::HybridMutex TestAllocatorStorage::M; +alignas(kMaxAlign) uint8_t TestAllocatorStorage::AllocatorStorage[kMaxSize]; +#else +struct TestAllocatorStorage { + static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS { + void *p = nullptr; + EXPECT_EQ(0, posix_memalign(&p, kMaxAlign, size)); + return p; + } + static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS { free(ptr); } +}; +#endif + +template +void *TestAllocator::operator new(size_t size) { + return TestAllocatorStorage::get(size); +} + +template +void TestAllocator::operator delete(void *ptr) { + TestAllocatorStorage::release(ptr); +} template struct ScudoCombinedTest : public Test { ScudoCombinedTest() { @@ -107,15 +166,60 @@ template struct ScudoCombinedTest : public Test { template using ScudoCombinedDeathTest = ScudoCombinedTest; +namespace scudo { +struct TestConditionVariableConfig { + static const bool MaySupportMemoryTagging = true; + template + using TSDRegistryT = + scudo::TSDRegistrySharedT; // Shared, max 8 TSDs. + + struct Primary { + using SizeClassMap = scudo::AndroidSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + static const scudo::uptr RegionSizeLog = 28U; + typedef scudo::u32 CompactPtrT; + static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const scudo::uptr GroupSizeLog = 20U; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; +#else + static const scudo::uptr RegionSizeLog = 18U; + static const scudo::uptr GroupSizeLog = 18U; + typedef scudo::uptr CompactPtrT; +#endif + static const scudo::s32 MinReleaseToOsIntervalMs = 1000; + static const scudo::s32 MaxReleaseToOsIntervalMs = 1000; + static const bool UseConditionVariable = true; +#if SCUDO_LINUX + using ConditionVariableT = scudo::ConditionVariableLinux; +#else + using ConditionVariableT = scudo::ConditionVariableDummy; +#endif + }; +#if SCUDO_CAN_USE_PRIMARY64 + template + using PrimaryT = scudo::SizeClassAllocator64; +#else + template + using PrimaryT = scudo::SizeClassAllocator32; +#endif + + struct Secondary { + template + using CacheT = scudo::MapAllocatorNoCache; + }; + template using SecondaryT = scudo::MapAllocator; +}; +} // namespace scudo + #if SCUDO_FUCHSIA #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, FuchsiaConfig) #else #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig) \ - SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig) + SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig) \ + SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig) #endif #define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \ @@ -170,6 +274,7 @@ void ScudoCombinedTest::BasicTest(scudo::uptr SizeLog) { } Allocator->printStats(); + Allocator->printFragmentationInfo(); } #define SCUDO_MAKE_BASIC_TEST(SizeLog) \ @@ -209,7 +314,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroContents) { void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true); EXPECT_NE(P, nullptr); for (scudo::uptr I = 0; I < Size; I++) - ASSERT_EQ((reinterpret_cast(P))[I], 0); + ASSERT_EQ((reinterpret_cast(P))[I], '\0'); memset(P, 0xaa, Size); Allocator->deallocate(P, Origin, Size); } @@ -227,7 +332,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroFill) { void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, false); EXPECT_NE(P, nullptr); for (scudo::uptr I = 0; I < Size; I++) - ASSERT_EQ((reinterpret_cast(P))[I], 0); + ASSERT_EQ((reinterpret_cast(P))[I], '\0'); memset(P, 0xaa, Size); Allocator->deallocate(P, Origin, Size); } @@ -286,7 +391,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeIncreasing) { // we preserve the data in the process. scudo::uptr Size = 16; void *P = Allocator->allocate(Size, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, Size); while (Size < TypeParam::Primary::SizeClassMap::MaxSize * 4) { void *NewP = Allocator->reallocate(P, Size * 2); @@ -308,7 +413,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeDecreasing) { scudo::uptr Size = TypeParam::Primary::SizeClassMap::MaxSize * 2; const scudo::uptr DataSize = 2048U; void *P = Allocator->allocate(Size, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, scudo::Min(Size, DataSize)); while (Size > 1U) { Size /= 2U; @@ -331,7 +436,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, ReallocateSame) { constexpr scudo::uptr ReallocSize = TypeParam::Primary::SizeClassMap::MaxSize - 64; void *P = Allocator->allocate(ReallocSize, Origin); - const char Marker = 0xab; + const char Marker = 'A'; memset(P, Marker, ReallocSize); for (scudo::sptr Delta = -32; Delta < 32; Delta += 8) { const scudo::uptr NewSize = @@ -388,7 +493,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) { disableDebuggerdMaybe(); void *P = Allocator->allocate(Size, Origin); Allocator->deallocate(P, Origin); - reinterpret_cast(P)[0] = 0xaa; + reinterpret_cast(P)[0] = 'A'; }, ""); EXPECT_DEATH( @@ -396,7 +501,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) { disableDebuggerdMaybe(); void *P = Allocator->allocate(Size, Origin); Allocator->deallocate(P, Origin); - reinterpret_cast(P)[Size - 1] = 0xaa; + reinterpret_cast(P)[Size - 1] = 'A'; }, ""); } @@ -408,15 +513,15 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) { if (Allocator->useMemoryTaggingTestOnly()) { // Check that disabling memory tagging works correctly. void *P = Allocator->allocate(2048, Origin); - EXPECT_DEATH(reinterpret_cast(P)[2048] = 0xaa, ""); + EXPECT_DEATH(reinterpret_cast(P)[2048] = 'A', ""); scudo::ScopedDisableMemoryTagChecks NoTagChecks; Allocator->disableMemoryTagging(); - reinterpret_cast(P)[2048] = 0xaa; + reinterpret_cast(P)[2048] = 'A'; Allocator->deallocate(P, Origin); P = Allocator->allocate(2048, Origin); EXPECT_EQ(scudo::untagPointer(P), P); - reinterpret_cast(P)[2048] = 0xaa; + reinterpret_cast(P)[2048] = 'A'; Allocator->deallocate(P, Origin); Allocator->releaseToOS(scudo::ReleaseToOS::Force); @@ -456,6 +561,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, CacheDrain) NO_THREAD_SAFETY_ANALYSIS { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); EXPECT_TRUE(!TSD->getCache().isEmpty()); TSD->getCache().drain(); EXPECT_TRUE(TSD->getCache().isEmpty()); @@ -480,6 +586,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ForceCacheDrain) NO_THREAD_SAFETY_ANALYSIS { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); EXPECT_TRUE(TSD->getCache().isEmpty()); EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U); EXPECT_TRUE(Allocator->getQuarantine()->isEmpty()); @@ -723,7 +830,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) { for (unsigned I = 0; I != Ptrs.size(); ++I) { Ptrs[I] = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true); for (scudo::uptr J = 0; J < Size; ++J) - ASSERT_EQ((reinterpret_cast(Ptrs[I]))[J], 0); + ASSERT_EQ((reinterpret_cast(Ptrs[I]))[J], '\0'); } } @@ -786,6 +893,7 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) { bool UnlockRequired; auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + TSD->assertLocked(/*BypassCheck=*/!UnlockRequired); TSD->getCache().drain(); Allocator->releaseToOS(scudo::ReleaseToOS::Force); @@ -793,41 +901,3 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) { #endif #endif - -#if SCUDO_LINUX - -SCUDO_TYPED_TEST(ScudoCombinedTest, SoftRssLimit) { - auto *Allocator = this->Allocator.get(); - Allocator->setRssLimitsTestOnly(1, 0, true); - - size_t Megabyte = 1024 * 1024; - size_t ChunkSize = 16; - size_t Error = 256; - - std::vector Ptrs; - for (size_t index = 0; index < Megabyte + Error; index += ChunkSize) { - void *Ptr = Allocator->allocate(ChunkSize, Origin); - Ptrs.push_back(Ptr); - } - - EXPECT_EQ(nullptr, Allocator->allocate(ChunkSize, Origin)); - - for (void *Ptr : Ptrs) - Allocator->deallocate(Ptr, Origin); -} - -SCUDO_TYPED_TEST(ScudoCombinedTest, HardRssLimit) { - auto *Allocator = this->Allocator.get(); - Allocator->setRssLimitsTestOnly(0, 1, false); - - size_t Megabyte = 1024 * 1024; - - EXPECT_DEATH( - { - disableDebuggerdMaybe(); - Allocator->allocate(Megabyte, Origin); - }, - ""); -} - -#endif diff --git a/Telegram/ThirdParty/scudo/tests/common_test.cpp b/Telegram/ThirdParty/scudo/tests/common_test.cpp index b1e55e80d..fff7c662a 100644 --- a/Telegram/ThirdParty/scudo/tests/common_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/common_test.cpp @@ -72,27 +72,4 @@ TEST(ScudoCommonTest, Zeros) { MemMap.unmap(MemMap.getBase(), Size); } -#if 0 -// This test is temorarily disabled because it may not work as expected. E.g., -// it doesn't dirty the pages so the pages may not be commited and it may only -// work on the single thread environment. As a result, this test is flaky and is -// impacting many test scenarios. -TEST(ScudoCommonTest, GetRssFromBuffer) { - constexpr int64_t AllocSize = 10000000; - constexpr int64_t Error = 3000000; - constexpr size_t Runs = 10; - - int64_t Rss = scudo::GetRSS(); - EXPECT_GT(Rss, 0); - - std::vector> Allocs(Runs); - for (auto &Alloc : Allocs) { - Alloc.reset(new char[AllocSize]()); - int64_t Prev = Rss; - Rss = scudo::GetRSS(); - EXPECT_LE(std::abs(Rss - AllocSize - Prev), Error); - } -} -#endif - } // namespace scudo diff --git a/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp b/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp new file mode 100644 index 000000000..caba1f64a --- /dev/null +++ b/Telegram/ThirdParty/scudo/tests/condition_variable_test.cpp @@ -0,0 +1,59 @@ +//===-- condition_variable_test.cpp -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "tests/scudo_unit_test.h" + +#include "common.h" +#include "condition_variable.h" +#include "mutex.h" + +#include + +template void simpleWaitAndNotifyAll() { + constexpr scudo::u32 NumThreads = 2; + constexpr scudo::u32 CounterMax = 1024; + std::thread Threads[NumThreads]; + + scudo::HybridMutex M; + ConditionVariableT CV; + CV.bindTestOnly(M); + scudo::u32 Counter = 0; + + for (scudo::u32 I = 0; I < NumThreads; ++I) { + Threads[I] = std::thread( + [&](scudo::u32 Id) { + do { + scudo::ScopedLock L(M); + if (Counter % NumThreads != Id && Counter < CounterMax) + CV.wait(M); + if (Counter >= CounterMax) { + break; + } else { + ++Counter; + CV.notifyAll(M); + } + } while (true); + }, + I); + } + + for (std::thread &T : Threads) + T.join(); + + EXPECT_EQ(Counter, CounterMax); +} + +TEST(ScudoConditionVariableTest, DummyCVWaitAndNotifyAll) { + simpleWaitAndNotifyAll(); +} + +#ifdef SCUDO_LINUX +TEST(ScudoConditionVariableTest, LinuxCVWaitAndNotifyAll) { + simpleWaitAndNotifyAll(); +} +#endif diff --git a/Telegram/ThirdParty/scudo/tests/memtag_test.cpp b/Telegram/ThirdParty/scudo/tests/memtag_test.cpp index 2488e0485..fd277f962 100644 --- a/Telegram/ThirdParty/scudo/tests/memtag_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/memtag_test.cpp @@ -120,7 +120,15 @@ TEST_F(MemtagTest, SelectRandomTag) { uptr Tags = 0; for (uptr I = 0; I < 100000; ++I) Tags = Tags | (1u << extractTag(selectRandomTag(Ptr, 0))); - EXPECT_EQ(0xfffeull, Tags); + // std::popcnt is C++20 + int PopCnt = 0; + while (Tags) { + PopCnt += Tags & 1; + Tags >>= 1; + } + // Random tags are not always very random, and this test is not about PRNG + // quality. Anything above half would be satisfactory. + EXPECT_GE(PopCnt, 8); } } diff --git a/Telegram/ThirdParty/scudo/tests/primary_test.cpp b/Telegram/ThirdParty/scudo/tests/primary_test.cpp index 6e580e4fe..181715117 100644 --- a/Telegram/ThirdParty/scudo/tests/primary_test.cpp +++ b/Telegram/ThirdParty/scudo/tests/primary_test.cpp @@ -8,6 +8,8 @@ #include "tests/scudo_unit_test.h" +#include "allocator_config.h" +#include "condition_variable.h" #include "primary32.h" #include "primary64.h" #include "size_class_map.h" @@ -104,6 +106,34 @@ template struct TestConfig4 { }; }; +// This is the only test config that enables the condition variable. +template struct TestConfig5 { + static const bool MaySupportMemoryTagging = true; + + struct Primary { + using SizeClassMap = SizeClassMapT; +#if defined(__mips__) + // Unable to allocate greater size on QEMU-user. + static const scudo::uptr RegionSizeLog = 23U; +#else + static const scudo::uptr RegionSizeLog = 24U; +#endif + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const scudo::uptr GroupSizeLog = 18U; + typedef scudo::u32 CompactPtrT; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + static const bool UseConditionVariable = true; +#if SCUDO_LINUX + using ConditionVariableT = scudo::ConditionVariableLinux; +#else + using ConditionVariableT = scudo::ConditionVariableDummy; +#endif + }; +}; + template