From 29233830dbfa2ec9b39eed3b924a71bf525e37bb Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Thu, 16 Mar 2023 12:38:22 -0700 Subject: [PATCH] initial revision of per tier stats --- cachelib/allocator/Cache.cpp | 9 +- cachelib/allocator/CacheAllocator-inl.h | 156 ++++++++++++++---- cachelib/allocator/CacheAllocator.h | 6 +- cachelib/allocator/CacheStats.cpp | 92 +++++++++-- cachelib/allocator/CacheStats.h | 32 +++- cachelib/allocator/CacheStatsInternal.h | 25 +-- .../tests/AllocatorMemoryTiersTest.cpp | 1 + .../tests/AllocatorMemoryTiersTest.h | 56 +++++++ cachelib/allocator/tests/TestBase-inl.h | 24 +++ cachelib/allocator/tests/TestBase.h | 5 + cachelib/cachebench/cache/Cache-inl.h | 31 ++-- cachelib/cachebench/cache/CacheStats.h | 93 +++++++---- 12 files changed, 411 insertions(+), 119 deletions(-) diff --git a/cachelib/allocator/Cache.cpp b/cachelib/allocator/Cache.cpp index 5c228ed7b5..9dc8b0c51f 100644 --- a/cachelib/allocator/Cache.cpp +++ b/cachelib/allocator/Cache.cpp @@ -235,17 +235,18 @@ void CacheBase::updateGlobalCacheStats(const std::string& statPrefix) const { statPrefix + "cache.size.configured", memStats.configuredRamCacheSize + memStats.nvmCacheSize); + //TODO: add specific per-tier counters const auto stats = getGlobalCacheStats(); counters_.updateDelta(statPrefix + "cache.alloc_attempts", - stats.allocAttempts); + std::accumulate(stats.allocAttempts.begin(), stats.allocAttempts.end(),0)); counters_.updateDelta(statPrefix + "cache.eviction_attempts", - stats.evictionAttempts); + std::accumulate(stats.evictionAttempts.begin(),stats.evictionAttempts.end(),0)); counters_.updateDelta(statPrefix + "cache.alloc_failures", - stats.allocFailures); + std::accumulate(stats.allocFailures.begin(),stats.allocFailures.end(),0)); counters_.updateDelta(statPrefix + "cache.invalid_allocs", stats.invalidAllocs); const std::string ramEvictionKey = statPrefix + "ram.evictions"; - counters_.updateDelta(ramEvictionKey, stats.numEvictions); + counters_.updateDelta(ramEvictionKey, std::accumulate(stats.numEvictions.begin(),stats.numEvictions.end(),0)); // get the new delta to see if uploading any eviction age stats or lifetime // stats makes sense. uint64_t ramEvictionDelta = counters_.getDelta(ramEvictionKey); diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index cee53dd156..954d533daa 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -418,8 +418,7 @@ CacheAllocator::allocateInternalTier(TierId tid, util::RollingLatencyTracker rollTracker{ (*stats_.classAllocLatency)[tid][pid][cid]}; - // TODO: per-tier - (*stats_.allocAttempts)[pid][cid].inc(); + (*stats_.allocAttempts)[tid][pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); @@ -445,12 +444,12 @@ CacheAllocator::allocateInternalTier(TierId tid, handle = acquire(new (memory) Item(key, size, creationTime, expiryTime)); if (handle) { handle.markNascent(); - (*stats_.fragmentationSize)[pid][cid].add( + (*stats_.fragmentationSize)[tid][pid][cid].add( util::getFragmentation(*this, *handle)); } } else { // failed to allocate memory. - (*stats_.allocFailures)[pid][cid].inc(); // TODO: per-tier + (*stats_.allocFailures)[tid][pid][cid].inc(); // wake up rebalancer if (poolRebalancer_) { poolRebalancer_->wakeUp(); @@ -522,16 +521,14 @@ CacheAllocator::allocateChainedItemInternal( util::RollingLatencyTracker rollTracker{ (*stats_.classAllocLatency)[tid][pid][cid]}; - // TODO: per-tier? Right now stats_ are not used in any public periodic - // worker - (*stats_.allocAttempts)[pid][cid].inc(); + (*stats_.allocAttempts)[tid][pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); if (memory == nullptr) { memory = findEviction(tid, pid, cid); } if (memory == nullptr) { - (*stats_.allocFailures)[pid][cid].inc(); + (*stats_.allocFailures)[tid][pid][cid].inc(); return WriteHandle{}; } @@ -543,7 +540,7 @@ CacheAllocator::allocateChainedItemInternal( if (child) { child.markNascent(); - (*stats_.fragmentationSize)[pid][cid].add( + (*stats_.fragmentationSize)[tid][pid][cid].add( util::getFragmentation(*this, *child)); } @@ -858,7 +855,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, stats_.perPoolEvictionAgeSecs_[allocInfo.poolId].trackValue(refreshTime); } - (*stats_.fragmentationSize)[allocInfo.poolId][allocInfo.classId].sub( + (*stats_.fragmentationSize)[tid][allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, it)); // Chained items can only end up in this place if the user has allocated @@ -941,7 +938,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, const auto childInfo = allocator_[tid]->getAllocInfo(static_cast(head)); - (*stats_.fragmentationSize)[childInfo.poolId][childInfo.classId].sub( + (*stats_.fragmentationSize)[tid][childInfo.poolId][childInfo.classId].sub( util::getFragmentation(*this, *head)); removeFromMMContainer(*head); @@ -1585,12 +1582,12 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { Item* candidate = nullptr; typename NvmCacheT::PutToken token; - mmContainer.withEvictionIterator([this, pid, cid, &candidate, &toRecycle, + mmContainer.withEvictionIterator([this, tid, pid, cid, &candidate, &toRecycle, &searchTries, &mmContainer, &lastTier, &token](auto&& itr) { if (!itr) { ++searchTries; - (*stats_.evictionAttempts)[pid][cid].inc(); + (*stats_.evictionAttempts)[tid][pid][cid].inc(); return; } @@ -1598,7 +1595,7 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { config_.evictionSearchTries > searchTries) && itr) { ++searchTries; - (*stats_.evictionAttempts)[pid][cid].inc(); + (*stats_.evictionAttempts)[tid][pid][cid].inc(); auto* toRecycle_ = itr.get(); auto* candidate_ = @@ -1698,6 +1695,7 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { XDCHECK(!candidate->isAccessible()); XDCHECK(candidate->getKey() == evictedToNext->getKey()); + (*stats_.numWritebacks)[tid][pid][cid].inc(); wakeUpWaiters(*candidate, std::move(evictedToNext)); } @@ -1707,9 +1705,9 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { // NULL. If `ref` == 0 then it means that we are the last holder of // that item. if (candidate->hasChainedItem()) { - (*stats_.chainedItemEvictions)[pid][cid].inc(); + (*stats_.chainedItemEvictions)[tid][pid][cid].inc(); } else { - (*stats_.regularItemEvictions)[pid][cid].inc(); + (*stats_.regularItemEvictions)[tid][pid][cid].inc(); } if (auto eventTracker = getEventTracker()) { @@ -2304,7 +2302,7 @@ bool CacheAllocator::recordAccessInMMContainer(Item& item, const auto tid = getTierId(item); const auto allocInfo = allocator_[tid]->getAllocInfo(static_cast(&item)); - (*stats_.cacheHits)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.cacheHits)[tid][allocInfo.poolId][allocInfo.classId].inc(); // track recently accessed items if needed if (UNLIKELY(config_.trackRecentItemsForDump)) { @@ -2773,6 +2771,8 @@ size_t CacheAllocator::getPoolSize(PoolId poolId) const { template PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { + //this pool ref is just used to get class ids, which will be the + //same across tiers const auto& pool = allocator_[currentTier()]->getPool(poolId); const auto& allocSizes = pool.getAllocSizes(); auto mpStats = pool.getStats(); @@ -2791,24 +2791,42 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { // TODO export evictions, numItems etc from compact cache directly. if (!isCompactCache) { for (const ClassId cid : classIds) { - uint64_t classHits = (*stats_.cacheHits)[poolId][cid].get(); - XDCHECK(mmContainers_[currentTier()][poolId][cid], - folly::sformat("Pid {}, Cid {} not initialized.", poolId, cid)); + uint64_t allocAttempts, evictionAttempts, allocFailures, + fragmentationSize, classHits, chainedItemEvictions, + regularItemEvictions, numWritebacks = 0; + MMContainerStat mmContainerStats; + for (TierId tid = 0; tid < getNumTiers(); tid++) { + allocAttempts += (*stats_.allocAttempts)[tid][poolId][cid].get(); + evictionAttempts += (*stats_.evictionAttempts)[tid][poolId][cid].get(); + allocFailures += (*stats_.allocFailures)[tid][poolId][cid].get(); + fragmentationSize += (*stats_.fragmentationSize)[tid][poolId][cid].get(); + classHits += (*stats_.cacheHits)[tid][poolId][cid].get(); + chainedItemEvictions += (*stats_.chainedItemEvictions)[tid][poolId][cid].get(); + regularItemEvictions += (*stats_.regularItemEvictions)[tid][poolId][cid].get(); + numWritebacks += (*stats_.numWritebacks)[tid][poolId][cid].get(); + mmContainerStats += getMMContainerStat(tid, poolId, cid); + XDCHECK(mmContainers_[tid][poolId][cid], + folly::sformat("Tid {}, Pid {}, Cid {} not initialized.", tid, poolId, cid)); + } cacheStats.insert( {cid, - {allocSizes[cid], (*stats_.allocAttempts)[poolId][cid].get(), - (*stats_.evictionAttempts)[poolId][cid].get(), - (*stats_.allocFailures)[poolId][cid].get(), - (*stats_.fragmentationSize)[poolId][cid].get(), classHits, - (*stats_.chainedItemEvictions)[poolId][cid].get(), - (*stats_.regularItemEvictions)[poolId][cid].get(), - getMMContainerStat(currentTier(), poolId, cid)}}); + {allocSizes[cid], + allocAttempts, + evictionAttempts, + allocFailures, + fragmentationSize, + classHits, + chainedItemEvictions, + regularItemEvictions, + numWritebacks, + mmContainerStats}}); totalHits += classHits; } } PoolStats ret; ret.isCompactCache = isCompactCache; + //pool name is also shared among tiers ret.poolName = allocator_[currentTier()]->getPoolName(poolId); ret.poolSize = pool.getPoolSize(); ret.poolUsableSize = pool.getPoolUsableSize(); @@ -2821,6 +2839,59 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { return ret; } +template +PoolStats CacheAllocator::getPoolStats(TierId tid, PoolId poolId) const { + const auto& pool = allocator_[tid]->getPool(poolId); + const auto& allocSizes = pool.getAllocSizes(); + auto mpStats = pool.getStats(); + const auto& classIds = mpStats.classIds; + + // check if this is a compact cache. + bool isCompactCache = false; + { + folly::SharedMutex::ReadHolder lock(compactCachePoolsLock_); + isCompactCache = isCompactCachePool_[poolId]; + } + + std::unordered_map cacheStats; + uint64_t totalHits = 0; + // cacheStats is only menaningful for pools that are not compact caches. + // TODO export evictions, numItems etc from compact cache directly. + if (!isCompactCache) { + for (const ClassId cid : classIds) { + uint64_t classHits = (*stats_.cacheHits)[tid][poolId][cid].get(); + XDCHECK(mmContainers_[tid][poolId][cid], + folly::sformat("Tid {}, Pid {}, Cid {} not initialized.", tid, poolId, cid)); + cacheStats.insert( + {cid, + {allocSizes[cid], + (*stats_.allocAttempts)[tid][poolId][cid].get(), + (*stats_.evictionAttempts)[tid][poolId][cid].get(), + (*stats_.allocFailures)[tid][poolId][cid].get(), + (*stats_.fragmentationSize)[tid][poolId][cid].get(), + classHits, + (*stats_.chainedItemEvictions)[tid][poolId][cid].get(), + (*stats_.regularItemEvictions)[tid][poolId][cid].get(), + (*stats_.numWritebacks)[tid][poolId][cid].get(), + getMMContainerStat(tid, poolId, cid)}}); + totalHits += classHits; + } + } + + PoolStats ret; + ret.isCompactCache = isCompactCache; + ret.poolName = allocator_[tid]->getPoolName(poolId); + ret.poolSize = pool.getPoolSize(); + ret.poolUsableSize = pool.getPoolUsableSize(); + ret.poolAdvisedSize = pool.getPoolAdvisedSize(); + ret.cacheStats = std::move(cacheStats); + ret.mpStats = std::move(mpStats); + ret.numPoolGetHits = totalHits; + ret.evictionAgeSecs = stats_.perPoolEvictionAgeSecs_[poolId].estimate(); + + return ret; +} + template ACStats CacheAllocator::getACStats(TierId tid, PoolId poolId, @@ -3072,7 +3143,7 @@ bool CacheAllocator::moveForSlabRelease( const auto allocInfo = allocator_[tid]->getAllocInfo(oldItem.getMemory()); allocator_[tid]->free(&oldItem); - (*stats_.fragmentationSize)[allocInfo.poolId][allocInfo.classId].sub( + (*stats_.fragmentationSize)[tid][allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, oldItem)); stats_.numMoveSuccesses.inc(); return true; @@ -3351,12 +3422,13 @@ void CacheAllocator::evictForSlabRelease( nvmCache_->put(*evicted, std::move(token)); } + const auto tid = getTierId(*evicted); const auto allocInfo = - allocator_[getTierId(*evicted)]->getAllocInfo(static_cast(evicted)); + allocator_[tid]->getAllocInfo(static_cast(evicted)); if (evicted->hasChainedItem()) { - (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.chainedItemEvictions)[tid][allocInfo.poolId][allocInfo.classId].inc(); } else { - (*stats_.regularItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.regularItemEvictions)[tid][allocInfo.poolId][allocInfo.classId].inc(); } stats_.numEvictionSuccesses.inc(); @@ -3579,8 +3651,13 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { for (PoolId pid : pools) { for (unsigned int cid = 0; cid < (*stats_.fragmentationSize)[pid].size(); ++cid) { + uint64_t fragmentationSize = 0; + for (TierId tid = 0; tid < getNumTiers(); tid++) { + fragmentationSize += (*stats_.fragmentationSize)[tid][pid][cid].get(); + } metadata_.fragmentationSize()[pid][static_cast(cid)] = - (*stats_.fragmentationSize)[pid][cid].get(); + fragmentationSize; + } if (isCompactCachePool_[pid]) { metadata_.compactCachePools()->push_back(pid); @@ -3826,8 +3903,19 @@ void CacheAllocator::initStats() { // deserialize the fragmentation size of each thread. for (const auto& pid : *metadata_.fragmentationSize()) { for (const auto& cid : pid.second) { - (*stats_.fragmentationSize)[pid.first][cid.first].set( - static_cast(cid.second)); + //in multi-tier we serialized as the sum - no way + //to get back so just divide the two for now + //TODO: proper multi-tier serialization + uint64_t total = static_cast(cid.second); + uint64_t part = total / getNumTiers(); + uint64_t sum = 0; + for (TierId tid = 1; tid < getNumTiers(); tid++) { + (*stats_.fragmentationSize)[tid][pid.first][cid.first].set(part); + sum += part; + } + uint64_t leftover = total - sum; + (*stats_.fragmentationSize)[0][pid.first][cid.first].set(leftover); + } } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 876931fd8f..05555eb48a 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1233,6 +1233,8 @@ class CacheAllocator : public CacheBase { // pool stats by pool id PoolStats getPoolStats(PoolId pid) const override final; + // pool stats by tier id and pool id + PoolStats getPoolStats(TierId tid, PoolId pid) const; // This can be expensive so it is not part of PoolStats PoolEvictionAgeStats getPoolEvictionAgeStats( @@ -2015,9 +2017,9 @@ auto& mmContainer = getMMContainer(tid, pid, cid); XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); if (candidate->hasChainedItem()) { - (*stats_.chainedItemEvictions)[pid][cid].inc(); + (*stats_.chainedItemEvictions)[tid][pid][cid].inc(); } else { - (*stats_.regularItemEvictions)[pid][cid].inc(); + (*stats_.regularItemEvictions)[tid][pid][cid].inc(); } // it's safe to recycle the item here as there are no more diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index b4770a3480..24fc8c6c1a 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -23,18 +23,21 @@ namespace cachelib { namespace detail { void Stats::init() { - cacheHits = std::make_unique(); - allocAttempts = std::make_unique(); - evictionAttempts = std::make_unique(); - fragmentationSize = std::make_unique(); - allocFailures = std::make_unique(); - chainedItemEvictions = std::make_unique(); - regularItemEvictions = std::make_unique(); + cacheHits = std::make_unique(); + allocAttempts = std::make_unique(); + evictionAttempts = std::make_unique(); + fragmentationSize = std::make_unique(); + allocFailures = std::make_unique(); + chainedItemEvictions = std::make_unique(); + regularItemEvictions = std::make_unique(); + numWritebacks = std::make_unique(); auto initToZero = [](auto& a) { - for (auto& s : a) { - for (auto& c : s) { + for (auto& t : a) { + for (auto& p : t) { + for (auto& c : p) { c.set(0); } + } } }; @@ -44,6 +47,7 @@ void Stats::init() { initToZero(*fragmentationSize); initToZero(*chainedItemEvictions); initToZero(*regularItemEvictions); + initToZero(*numWritebacks); classAllocLatency = std::make_unique(); } @@ -116,20 +120,43 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.nvmEvictionSecondsToExpiry = this->nvmEvictionSecondsToExpiry_.estimate(); ret.nvmPutSize = this->nvmPutSize_.estimate(); - auto accum = [](const PerPoolClassAtomicCounters& c) { - uint64_t sum = 0; - for (const auto& x : c) { - for (const auto& v : x) { - sum += v.get(); - } + auto accum = [](const PerTierPerPoolClassAtomicCounters& t) { + std::vector stat; + for (const auto& c : t) { + uint64_t sum = 0; + for (const auto& x : c) { + for (const auto& v : x) { + sum += v.get(); + } + } + stat.push_back(sum); + } + return stat; + }; + + auto accumTL = [](const PerTierPerPoolClassTLCounters& t) { + std::vector stat; + for (const auto& c : t) { + uint64_t sum = 0; + for (const auto& x : c) { + for (const auto& v : x) { + sum += v.get(); + } + } + stat.push_back(sum); } - return sum; + return stat; }; ret.allocAttempts = accum(*allocAttempts); ret.evictionAttempts = accum(*evictionAttempts); ret.allocFailures = accum(*allocFailures); - ret.numEvictions = accum(*chainedItemEvictions); - ret.numEvictions += accum(*regularItemEvictions); + auto chainedEvictions = accum(*chainedItemEvictions); + auto regularEvictions = accum(*regularItemEvictions); + for (TierId tid = 0; tid < chainedEvictions.size(); tid++) { + ret.numEvictions.push_back(chainedEvictions[tid] + regularEvictions[tid]); + } + ret.numWritebacks = accum(*numWritebacks); + ret.numCacheHits = accumTL(*cacheHits); ret.invalidAllocs = invalidAllocs.get(); ret.numRefcountOverflow = numRefcountOverflow.get(); @@ -145,6 +172,18 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { } // namespace detail +MMContainerStat& MMContainerStat::operator+=(const MMContainerStat& other) { + + size += other.size; + oldestTimeSec = std::min(oldestTimeSec,other.oldestTimeSec); + lruRefreshTime = std::max(lruRefreshTime,other.lruRefreshTime); + numHotAccesses += other.numHotAccesses; + numColdAccesses += other.numColdAccesses; + numWarmAccesses += other.numWarmAccesses; + numTailAccesses += other.numTailAccesses; + return *this; +} + PoolStats& PoolStats::operator+=(const PoolStats& other) { auto verify = [](bool isCompatible) { if (!isCompatible) { @@ -182,6 +221,7 @@ PoolStats& PoolStats::operator+=(const PoolStats& other) { d.allocFailures += s.allocFailures; d.fragmentationSize += s.fragmentationSize; d.numHits += s.numHits; + d.numWritebacks += s.numWritebacks; d.chainedItemEvictions += s.chainedItemEvictions; d.regularItemEvictions += s.regularItemEvictions; } @@ -237,6 +277,14 @@ uint64_t PoolStats::numEvictions() const noexcept { return n; } +uint64_t PoolStats::numWritebacks() const noexcept { + uint64_t n = 0; + for (const auto& s : cacheStats) { + n += s.second.numWritebacks; + } + return n; +} + uint64_t PoolStats::numItems() const noexcept { uint64_t n = 0; for (const auto& s : cacheStats) { @@ -245,6 +293,14 @@ uint64_t PoolStats::numItems() const noexcept { return n; } +uint64_t PoolStats::numHits() const noexcept { + uint64_t n = 0; + for (const auto& s : cacheStats) { + n += s.second.numHits; + } + return n; +} + uint64_t PoolStats::numAllocFailures() const { uint64_t n = 0; for (const auto& s : cacheStats) { diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 46c051be14..91daf6f80b 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -94,6 +94,9 @@ struct MMContainerStat { uint64_t numColdAccesses; uint64_t numWarmAccesses; uint64_t numTailAccesses; + + // aggregate stats together (accross tiers) + MMContainerStat& operator+=(const MMContainerStat& other); }; // cache related stats for a given allocation class. @@ -114,13 +117,16 @@ struct CacheStat { uint64_t fragmentationSize{0}; // number of hits for this container. - uint64_t numHits; + uint64_t numHits{0}; // number of evictions from this class id that was of a chained item - uint64_t chainedItemEvictions; + uint64_t chainedItemEvictions{0}; // number of regular items that were evicted from this classId - uint64_t regularItemEvictions; + uint64_t regularItemEvictions{0}; + + // number of items that are moved to next tier + uint64_t numWritebacks{0}; // the stats from the mm container MMContainerStat containerStat; @@ -199,6 +205,9 @@ struct PoolStats { // number of evictions for this pool uint64_t numEvictions() const noexcept; + // number of writebacks for this pool + uint64_t numWritebacks() const noexcept; + // number of all items in this pool uint64_t numItems() const noexcept; @@ -208,6 +217,9 @@ struct PoolStats { // total number of allocations currently in this pool uint64_t numActiveAllocs() const noexcept; + // number of hits for an alloc class in this pool + uint64_t numHits() const noexcept; + // number of hits for an alloc class in this pool uint64_t numHitsForClass(ClassId cid) const { return cacheStats.at(cid).numHits; @@ -442,16 +454,22 @@ struct GlobalCacheStats { uint64_t numNvmItemRemovedSetSize{0}; // number of attempts to allocate an item - uint64_t allocAttempts{0}; + std::vector allocAttempts; // number of eviction attempts - uint64_t evictionAttempts{0}; + std::vector evictionAttempts; // number of failures to allocate an item due to internal error - uint64_t allocFailures{0}; + std::vector allocFailures; // number of evictions across all the pools in the cache. - uint64_t numEvictions{0}; + std::vector numEvictions; + + // number of writebacks across all the pools in the cache. + std::vector numWritebacks; + + // number of hits per tier across all the pools in the cache. + std::vector numCacheHits; // number of allocation attempts with invalid input params. uint64_t invalidAllocs{0}; diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index 19a15fbbd4..ef41ea7bbc 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -212,23 +212,26 @@ struct Stats { // we're currently writing into flash. mutable util::PercentileStats nvmPutSize_; - using PerPoolClassAtomicCounters = + using PerTierPerPoolClassAtomicCounters = std::array< std::array, - MemoryPoolManager::kMaxPools>; + MemoryPoolManager::kMaxPools>, + CacheBase::kMaxTiers>; // count of a stat for a specific allocation class - using PerPoolClassTLCounters = + using PerTierPerPoolClassTLCounters = std::array< std::array, - MemoryPoolManager::kMaxPools>; + MemoryPoolManager::kMaxPools>, + CacheBase::kMaxTiers>; // hit count for every alloc class in every pool - std::unique_ptr cacheHits{}; - std::unique_ptr allocAttempts{}; - std::unique_ptr evictionAttempts{}; - std::unique_ptr allocFailures{}; - std::unique_ptr fragmentationSize{}; - std::unique_ptr chainedItemEvictions{}; - std::unique_ptr regularItemEvictions{}; + std::unique_ptr cacheHits{}; + std::unique_ptr allocAttempts{}; + std::unique_ptr evictionAttempts{}; + std::unique_ptr allocFailures{}; + std::unique_ptr fragmentationSize{}; + std::unique_ptr chainedItemEvictions{}; + std::unique_ptr regularItemEvictions{}; + std::unique_ptr numWritebacks{}; using PerTierPoolClassRollingStats = std::array< std::array, diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index ad845d94df..5df2b1972e 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -25,6 +25,7 @@ using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; // TODO(MEMORY_TIER): add more tests with different eviction policies TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidStats) { this->testMultiTiersValidStats(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index c724cd9617..faa9d915e2 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -93,6 +93,62 @@ class AllocatorMemoryTiersTest : public AllocatorTest { ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } + void testMultiTiersValidStats() { + typename AllocatorT::Config config; + size_t nSlabs = 20; + config.setCacheSize(nSlabs * Slab::kSize); + config.enableCachePersistence("/tmp"); + ASSERT_NO_THROW(config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0")), + MemoryTierCacheConfig::fromShm().setRatio(2).setMemBind( + std::string("0"))})); + + auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + size_t keyLen = 8; + auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + std::vector valsize = {1000}; + std::vector itemCount; + std::vector evictCount; + for (uint32_t tid = 0; tid < 2; tid++) { + this->fillUpPoolUntilEvictions(*alloc, tid, pool, valsize, keyLen); + auto stats = alloc->getPoolStats(tid, pool); + const auto& classIds = stats.mpStats.classIds; + uint32_t prev = 0; + ClassId cid = 0; + for (const ClassId c : classIds) { + uint32_t currSize = stats.cacheStats[c].allocSize; + if (prev <= valsize[0] && valsize[0] <= currSize) { + cid = c; + break; + } + prev = currSize; + } + + std::cout << "Tid: " << tid << " cid: " << static_cast(cid) + << " items: " << stats.cacheStats[cid].numItems() + << " evicts: " << stats.cacheStats[cid].numEvictions() + << std::endl; + ASSERT_GE(stats.cacheStats[cid].numItems(), 1); + ASSERT_EQ(stats.cacheStats[cid].numEvictions(), 1); + itemCount.push_back(stats.cacheStats[cid].numItems()); + evictCount.push_back(stats.cacheStats[cid].numEvictions()); + //first tier should have some writebacks to second tier + //second tier should not have any writebacks since it + //is last memory tier + if (tid == 0) { + ASSERT_EQ(stats.cacheStats[cid].numWritebacks, 1); + } else { + ASSERT_EQ(0, stats.cacheStats[cid].numWritebacks); + } + } + for (uint32_t tid = 1; tid < 2; tid++) { + ASSERT_NE(itemCount[tid],itemCount[tid-1]); + ASSERT_EQ(evictCount[tid],evictCount[tid-1]); + } + } + void testMultiTiersBackgroundMovers() { typename AllocatorT::Config config; config.setCacheSize(10 * Slab::kSize); diff --git a/cachelib/allocator/tests/TestBase-inl.h b/cachelib/allocator/tests/TestBase-inl.h index 4d45e981bc..79dc6f44be 100644 --- a/cachelib/allocator/tests/TestBase-inl.h +++ b/cachelib/allocator/tests/TestBase-inl.h @@ -98,6 +98,30 @@ void AllocatorTest::fillUpPoolUntilEvictions( } while (allocs != 0); } +template +void AllocatorTest::fillUpPoolUntilEvictions( + AllocatorT& alloc, + TierId tid, + PoolId poolId, + const std::vector& sizes, + unsigned int keyLen) { + unsigned int allocs = 0; + do { + allocs = 0; + for (const auto size : sizes) { + const auto key = getRandomNewKey(alloc, keyLen); + ASSERT_EQ(alloc.find(key), nullptr); + const size_t prev = alloc.getPoolByTid(poolId, tid).getCurrentAllocSize(); + auto handle = util::allocateAccessible(alloc, poolId, key, size); + if (handle && prev != alloc.getPoolByTid(poolId, tid).getCurrentAllocSize()) { + // this means we did not cause an eviction. + ASSERT_GE(handle->getSize(), size); + allocs++; + } + } + } while (allocs != 0); +} + template void AllocatorTest::testAllocWithoutEviction( AllocatorT& alloc, diff --git a/cachelib/allocator/tests/TestBase.h b/cachelib/allocator/tests/TestBase.h index 54032e3257..858da2bb95 100644 --- a/cachelib/allocator/tests/TestBase.h +++ b/cachelib/allocator/tests/TestBase.h @@ -69,6 +69,11 @@ class AllocatorTest : public SlabAllocatorTestBase { PoolId pid, const std::vector& sizes, unsigned int keyLen); + void fillUpPoolUntilEvictions(AllocatorT& alloc, + TierId tid, + PoolId pid, + const std::vector& sizes, + unsigned int keyLen); void fillUpOneSlab(AllocatorT& alloc, PoolId poolId, const uint32_t size, diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index a73763bcc4..2a6861c8ec 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -622,18 +622,25 @@ double Cache::getNvmBytesWritten() const { template Stats Cache::getStats() const { - PoolStats aggregate = cache_->getPoolStats(pools_[0]); - auto usageFraction = - 1.0 - (static_cast(aggregate.freeMemoryBytes())) / - aggregate.poolUsableSize; + Stats ret; - ret.poolUsageFraction.push_back(usageFraction); - for (size_t pid = 1; pid < pools_.size(); pid++) { - auto poolStats = cache_->getPoolStats(static_cast(pid)); - usageFraction = 1.0 - (static_cast(poolStats.freeMemoryBytes())) / - poolStats.poolUsableSize; - ret.poolUsageFraction.push_back(usageFraction); - aggregate += poolStats; + for (TierId tid = 0; tid < cache_->getNumTiers(); tid++) { + PoolStats aggregate = cache_->getPoolStats(tid,pools_[0]); + auto usageFraction = + 1.0 - (static_cast(aggregate.freeMemoryBytes())) / + aggregate.poolUsableSize; + ret.poolUsageFraction[tid].push_back(usageFraction); + for (size_t pid = 1; pid < pools_.size(); pid++) { + auto poolStats = cache_->getPoolStats(tid, static_cast(pid)); + usageFraction = 1.0 - (static_cast(poolStats.freeMemoryBytes())) / + poolStats.poolUsableSize; + ret.poolUsageFraction[tid].push_back(usageFraction); + aggregate += poolStats; + } + ret.numEvictions.push_back(aggregate.numEvictions()); + ret.numWritebacks.push_back(aggregate.numWritebacks()); + ret.numCacheHits.push_back(aggregate.numHits()); + ret.numItems.push_back(aggregate.numItems()); } std::map>> allocationClassStats{}; @@ -666,8 +673,6 @@ Stats Cache::getStats() const { ret.backgndPromoStats.nTraversals = cacheStats.promotionStats.runCount; - ret.numEvictions = aggregate.numEvictions(); - ret.numItems = aggregate.numItems(); ret.evictAttempts = cacheStats.evictionAttempts; ret.allocAttempts = cacheStats.allocAttempts; ret.allocFailures = cacheStats.allocFailures; diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 993f3845a3..4034e1249b 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -52,15 +52,18 @@ struct BackgroundPromotionStats { struct Stats { BackgroundEvictionStats backgndEvicStats; BackgroundPromotionStats backgndPromoStats; + ReaperStats reaperStats; - uint64_t numEvictions{0}; - uint64_t numItems{0}; + std::vector numEvictions; + std::vector numWritebacks; + std::vector numCacheHits; + std::vector numItems; - uint64_t evictAttempts{0}; - uint64_t allocAttempts{0}; - uint64_t allocFailures{0}; + std::vector evictAttempts{0}; + std::vector allocAttempts{0}; + std::vector allocFailures{0}; - std::vector poolUsageFraction; + std::map> poolUsageFraction; uint64_t numCacheGets{0}; uint64_t numCacheGetMiss{0}; @@ -143,20 +146,31 @@ struct Stats { void render(std::ostream& out) const { auto totalMisses = getTotalMisses(); const double overallHitRatio = invertPctFn(totalMisses, numCacheGets); - out << folly::sformat("Items in RAM : {:,}", numItems) << std::endl; - out << folly::sformat("Items in NVM : {:,}", numNvmItems) << std::endl; - - out << folly::sformat("Alloc Attempts: {:,} Success: {:.2f}%", - allocAttempts, - invertPctFn(allocFailures, allocAttempts)) - << std::endl; - out << folly::sformat( - "Evict Attempts: {:,} Success: {:.2f}%", - evictAttempts, - invertPctFn(evictAttempts - numEvictions, evictAttempts)) - << std::endl; - out << folly::sformat("RAM Evictions : {:,}", numEvictions) << std::endl; - + const auto nTiers = numItems.size(); + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Items in Tier {} : {:,}", tid, numItems[tid]) << std::endl; + } + out << folly::sformat("Items in NVM : {:,}", numNvmItems) << std::endl; + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Tier {} Alloc Attempts: {:,} Success: {:.2f}%", + tid, + allocAttempts[tid], + invertPctFn(allocFailures[tid], allocAttempts[tid])) + << std::endl; + } + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat( + "Tier {} Evict Attempts: {:,} Success: {:.2f}%", + tid, + evictAttempts[tid], + invertPctFn(evictAttempts[tid] - numEvictions[tid], evictAttempts[tid])) + << std::endl; + } + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Tier {} Evictions : {:,} Writebacks: {:,} Success: {:.2f}%", + tid, numEvictions[tid], numWritebacks[tid], + invertPctFn(numEvictions[tid] - numWritebacks[tid], numEvictions[tid])) << std::endl; + } auto foreachAC = [&](auto &map, auto cb) { for (auto &tidStats : map) { for (auto &pidStat : tidStats.second) { @@ -167,10 +181,16 @@ struct Stats { } }; - for (auto pid = 0U; pid < poolUsageFraction.size(); pid++) { - out << folly::sformat("Fraction of pool {:,} used : {:.2f}", pid, - poolUsageFraction[pid]) - << std::endl; + for (auto entry : poolUsageFraction) { + auto tid = entry.first; + auto usageFraction = entry.second; + for (auto pid = 0U; pid < usageFraction.size(); pid++) { + out << folly::sformat("Tier {} fraction of pool {:,} used : {:.2f}", + tid, + pid, + usageFraction[pid]) + << std::endl; + } } if (FLAGS_report_ac_memory_usage_stats != "") { @@ -212,8 +232,8 @@ struct Stats { // If the pool is not full, extrapolate usageFraction for AC assuming it // will grow at the same rate. This value will be the same for all ACs. - auto acUsageFraction = (poolUsageFraction[pid] < 1.0) - ? poolUsageFraction[pid] + const auto acUsageFraction = (poolUsageFraction.at(tid)[pid] < 1.0) + ? poolUsageFraction.at(tid)[pid] : stats.usageFraction(); out << folly::sformat( @@ -235,7 +255,11 @@ struct Stats { out << folly::sformat("Cache Gets : {:,}", numCacheGets) << std::endl; out << folly::sformat("Hit Ratio : {:6.2f}%", overallHitRatio) << std::endl; - + for (TierId tid = 0; tid < numCacheHits.size(); tid++) { + double tierHitRatio = pctFn(numCacheHits[tid],numCacheGets); + out << folly::sformat("Tier {} Hit Ratio : {:6.2f}%", tid, tierHitRatio) + << std::endl; + } if (FLAGS_report_api_latency) { auto printLatencies = [&out](folly::StringPiece cat, @@ -277,6 +301,14 @@ struct Stats { }); } + if (reaperStats.numReapedItems > 0) { + + out << folly::sformat("Reaper reaped: {:,} visited: {:,} traversals: {:,} avg traversal time: {:,}", + reaperStats.numReapedItems,reaperStats.numVisitedItems, + reaperStats.numTraversals,reaperStats.avgTraversalTimeMs) + << std::endl; + } + if (numNvmGets > 0 || numNvmDeletes > 0 || numNvmPuts > 0) { const double ramHitRatio = invertPctFn(numCacheGetMiss, numCacheGets); const double nvmHitRatio = invertPctFn(numNvmGetMiss, numNvmGets); @@ -412,8 +444,8 @@ struct Stats { } if (numCacheEvictions > 0) { - out << folly::sformat("Total eviction executed {}", numCacheEvictions) - << std::endl; + out << folly::sformat("Total evictions executed {:,}", numCacheEvictions) + << std::endl; } } @@ -468,7 +500,8 @@ struct Stats { }; auto totalMisses = getTotalMisses(); - counters["num_items"] = numItems; + //TODO: per tier + counters["num_items"] = std::accumulate(numItems.begin(),numItems.end(),0); counters["num_nvm_items"] = numNvmItems; counters["hit_rate"] = calcInvertPctFn(totalMisses, numCacheGets);