diff --git a/cachelib/allocator/BackgroundEvictor-inl.h b/cachelib/allocator/BackgroundEvictor-inl.h new file mode 100644 index 0000000000..bdf72be209 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor-inl.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +namespace facebook { +namespace cachelib { + + +template +BackgroundEvictor::BackgroundEvictor(Cache& cache, + std::shared_ptr strategy, + unsigned int tid) + : cache_(cache), + strategy_(strategy), + tid_(tid) { +} + +template +BackgroundEvictor::~BackgroundEvictor() { stop(std::chrono::seconds(0)); } + +template +void BackgroundEvictor::work() { + try { + if (!tasks_.empty()) { + while (auto entry = tasks_.try_dequeue()) { + auto [pid, cid] = entry.value(); + auto batch = strategy_->calculateBatchSize(cache_, tid_, pid, cid); + stats.evictionSize.add(batch); + auto evicted = BackgroundEvictorAPIWrapper::traverseAndEvictItems(cache_, + tid_,pid,cid,batch); + stats.numEvictedItemsFromSchedule.inc(); + stats.numTraversals.inc(); + } + } else { + for (const auto pid : cache_.getRegularPoolIds()) { + //check if the pool is full - probably should be if tier is full + if (cache_.getPoolByTid(pid,tid_).allSlabsAllocated()) { + checkAndRun(pid); + } + } + } + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundEvictor interrupted due to exception: {}", ex.what()); + } +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundEvictor::checkAndRun(PoolId pid) { + const auto& mpStats = cache_.getPoolByTid(pid,tid_).getStats(); + unsigned int evictions = 0; + unsigned int classes = 0; + for (auto& cid : mpStats.classIds) { + classes++; + auto batch = strategy_->calculateBatchSize(cache_,tid_,pid,cid); + if (!batch) { + continue; + } + + stats.evictionSize.add(batch); + //try evicting BATCH items from the class in order to reach free target + auto evicted = + BackgroundEvictorAPIWrapper::traverseAndEvictItems(cache_, + tid_,pid,cid,batch); + + evictions += evicted; + const size_t cid_id = (size_t)mpStats.acStats.at(cid).allocSize; + auto it = evictions_per_class_.find(cid_id); + if (it != evictions_per_class_.end()) { + it->second += evicted; + } else { + evictions_per_class_[cid_id] = 0; + } + } + stats.numEvictedItems.add(evictions); + stats.numClasses.add(classes); + stats.numTraversals.inc(); +} + +template +BackgroundEvictionStats BackgroundEvictor::getStats() const noexcept { + BackgroundEvictionStats evicStats; + evicStats.numEvictedItems = stats.numEvictedItems.get(); + evicStats.numEvictedItemsFromSchedule = stats.numEvictedItemsFromSchedule.get(); + evicStats.numTraversals = stats.numTraversals.get(); + evicStats.numClasses = stats.numClasses.get(); + evicStats.evictionSize = stats.evictionSize.get(); + + return evicStats; +} + +template +std::map BackgroundEvictor::getClassStats() const noexcept { + return evictions_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundEvictor.h b/cachelib/allocator/BackgroundEvictor.h new file mode 100644 index 0000000000..7cecb04ba1 --- /dev/null +++ b/cachelib/allocator/BackgroundEvictor.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/PeriodicWorker.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" +#include "cachelib/common/AtomicCounter.h" + + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the eviction. +template +struct BackgroundEvictorAPIWrapper { + + static size_t traverseAndEvictItems(C& cache, + unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + return cache.traverseAndEvictItems(tid,pid,cid,batch); + } +}; + +struct BackgroundEvictorStats { + // items evicted + AtomicCounter numEvictedItems{0}; + + // items evicted from schedule + AtomicCounter numEvictedItemsFromSchedule; + + // traversals + AtomicCounter numTraversals{0}; + + // total number of classes + AtomicCounter numClasses{0}; + + // item eviction size + AtomicCounter evictionSize{0}; +}; + +// Periodic worker that evicts items from tiers in batches +// The primary aim is to reduce insertion times for new items in the +// cache +template +class BackgroundEvictor : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param target_free the target amount of memory to keep free in + // this tier + // @param tier id memory tier to perform eviction on + BackgroundEvictor(Cache& cache, + std::shared_ptr strategy, + unsigned int tid); + + ~BackgroundEvictor() override; + + void schedule(size_t pid, size_t cid) { + tasks_.enqueue(std::make_pair(pid,cid)); + } + + BackgroundEvictionStats getStats() const noexcept; + + std::map getClassStats() const noexcept; + + private: + // cache allocator's interface for evicting + + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + unsigned int tid_; + folly::UMPSCQueue,true> tasks_; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(PoolId pid); + + BackgroundEvictorStats stats; + + std::map evictions_per_class_; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundEvictor-inl.h" diff --git a/cachelib/allocator/BackgroundEvictorStrategy.h b/cachelib/allocator/BackgroundEvictorStrategy.h new file mode 100644 index 0000000000..00dd3e878b --- /dev/null +++ b/cachelib/allocator/BackgroundEvictorStrategy.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class BackgroundEvictorStrategy { + +public: + virtual size_t calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid ) = 0; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index b64d48d86f..b250923ba7 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -35,8 +35,10 @@ add_library (cachelib_allocator CCacheManager.cpp ContainerTypes.cpp FreeMemStrategy.cpp + FreeThresholdStrategy.cpp HitsPerSlabStrategy.cpp LruTailAgeStrategy.cpp + KeepFreeStrategy.cpp MarginalHitsOptimizeStrategy.cpp MarginalHitsStrategy.cpp memory/AllocationClass.cpp diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index c4a48506d3..41536c3f13 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -93,6 +93,12 @@ class CacheBase { // // @param poolId The pool id to query virtual const MemoryPool& getPool(PoolId poolId) const = 0; + + // Get the reference to a memory pool using a tier id, for stats purposes + // + // @param poolId The pool id to query + // @param tierId The tier of the pool id + virtual const MemoryPool& getPoolByTid(PoolId poolId, TierId tid) const = 0; // Get Pool specific stats (regular pools). This includes stats from the // Memory Pool and also the cache. diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index c8c11c77f5..297fc29f35 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -325,6 +325,12 @@ void CacheAllocator::initWorkers() { config_.poolOptimizeStrategy, config_.ccacheOptimizeStepSizePercent); } + + if (config_.backgroundEvictorEnabled()) { + startNewBackgroundEvictor(config_.backgroundEvictorInterval, + config_.backgroundEvictorStrategy, + 0); //right now default to tier 0); + } } template @@ -376,6 +382,14 @@ CacheAllocator::allocateInternalTier(TierId tid, // Should we support eviction between memory tiers (e.g. from DRAM to PMEM)? if (memory == nullptr && !config_.disableEviction) { memory = findEviction(tid, pid, cid); + + if (backgroundEvictor_ && config_.scheduleEviction) { + backgroundEvictor_->schedule(pid,cid); + } + + if (backgroundEvictor_ && config_.wakeupBgEvictor) { + backgroundEvictor_->wakeUp(); + } } ItemHandle handle; @@ -3381,6 +3395,7 @@ bool CacheAllocator::stopWorkers(std::chrono::seconds timeout) { success &= stopPoolResizer(timeout); success &= stopMemMonitor(timeout); success &= stopReaper(timeout); + success &= stopBackgroundEvictor(timeout); return success; } @@ -3661,6 +3676,7 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.nvmUpTime = currTime - getNVMCacheCreationTime(); ret.reaperStats = getReaperStats(); + ret.evictionStats = getBackgroundEvictorStats(); ret.numActiveHandles = getNumActiveHandles(); return ret; @@ -3759,6 +3775,7 @@ bool CacheAllocator::startNewPoolRebalancer( freeAllocThreshold); } + template bool CacheAllocator::startNewPoolResizer( std::chrono::milliseconds interval, @@ -3796,6 +3813,14 @@ bool CacheAllocator::startNewReaper( return startNewWorker("Reaper", reaper_, interval, reaperThrottleConfig); } +template +bool CacheAllocator::startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + unsigned int tid ) { + return startNewWorker("BackgroundEvictor", backgroundEvictor_, interval, strategy, tid); +} + template bool CacheAllocator::stopPoolRebalancer( std::chrono::seconds timeout) { @@ -3823,6 +3848,12 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { return stopWorker("Reaper", reaper_, timeout); } +template +bool CacheAllocator::stopBackgroundEvictor( + std::chrono::seconds timeout) { + return stopWorker("BackgroundEvictor", backgroundEvictor_, timeout); +} + template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 319e66a626..4e62d34923 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -36,7 +36,7 @@ #include #include #pragma GCC diagnostic pop - +#include "cachelib/allocator/BackgroundEvictor.h" #include "cachelib/allocator/CCacheManager.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheAllocatorConfig.h" @@ -945,6 +945,9 @@ class CacheAllocator : public CacheBase { // @param reaperThrottleConfig throttling config bool startNewReaper(std::chrono::milliseconds interval, util::Throttler::Config reaperThrottleConfig); + + bool startNewBackgroundEvictor(std::chrono::milliseconds interval, + std::shared_ptr strategy, unsigned int tid); // Stop existing workers with a timeout bool stopPoolRebalancer(std::chrono::seconds timeout = std::chrono::seconds{ @@ -954,6 +957,7 @@ class CacheAllocator : public CacheBase { 0}); bool stopMemMonitor(std::chrono::seconds timeout = std::chrono::seconds{0}); bool stopReaper(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundEvictor(std::chrono::seconds timeout = std::chrono::seconds{0}); // Set pool optimization to either true or false // @@ -988,6 +992,10 @@ class CacheAllocator : public CacheBase { const MemoryPool& getPool(PoolId pid) const override final { return allocator_[currentTier()]->getPool(pid); } + + const MemoryPool& getPoolByTid(PoolId pid, TierId tid) const override final { + return allocator_[tid]->getPool(pid); + } // calculate the number of slabs to be advised/reclaimed in each pool PoolAdviseReclaimData calcNumSlabsToAdviseReclaim() override final { @@ -1034,6 +1042,17 @@ class CacheAllocator : public CacheBase { auto stats = reaper_ ? reaper_->getStats() : ReaperStats{}; return stats; } + + // returns the background evictor + BackgroundEvictionStats getBackgroundEvictorStats() const { + auto stats = backgroundEvictor_ ? backgroundEvictor_->getStats() : BackgroundEvictionStats{}; + return stats; + } + + std::map getBackgroundEvictorClassStats() const { + auto stats = backgroundEvictor_ ? backgroundEvictor_->getClassStats() : std::map(); + return stats; + } // return the LruType of an item typename MMType::LruType getItemLruType(const Item& item) const; @@ -1752,6 +1771,61 @@ class CacheAllocator : public CacheBase { folly::annotate_ignore_thread_sanitizer_guard g(__FILE__, __LINE__); allocator_[currentTier()]->forEachAllocation(std::forward(f)); } + + // exposed for the background evictor to iterate through the memory and evict + // in batch. This should improve insertion path for tiered memory config + size_t traverseAndEvictItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t evictions = 0; + auto itr = mmContainer.getEvictionIterator(); + + while (evictions < batch && itr) { + Item* candidate = itr.get(); + if (candidate == NULL) { + break; + } + // for chained items, the ownership of the parent can change. We try to + // evict what we think as parent and see if the eviction of parent + // recycles the child we intend to. + + ItemHandle toReleaseHandle = tryEvictToNextMemoryTier(tid, pid, itr); + if (!toReleaseHandle) { + ++itr; + } else { + if (toReleaseHandle->hasChainedItem()) { + (*stats_.chainedItemEvictions)[pid][cid].inc(); + } else { + (*stats_.regularItemEvictions)[pid][cid].inc(); + } + ++evictions; + + // we must be the last handle and for chained items, this will be + // the parent. + XDCHECK(toReleaseHandle.get() == candidate || candidate->isChainedItem()); + XDCHECK_EQ(1u, toReleaseHandle->getRefCount()); + + // We manually release the item here because we don't want to + // invoke the Item Handle's destructor which will be decrementing + // an already zero refcount, which will throw exception + auto& itemToRelease = *toReleaseHandle.release(); + + // Decrementing the refcount because we will call releaseBackToAllocator + const auto ref = decRef(itemToRelease); + XDCHECK_EQ(0u, ref); + + // check if by releasing the item we intend to, we actually + // recycle the candidate. + const auto res = releaseBackToAllocator(itemToRelease, RemoveContext::kEviction, + /* isNascent */ true); + XDCHECK(res == ReleaseRes::kReleased); + } + } + + // Invalidate iterator since later on we may use this mmContainer + // again, which cannot be done unless we drop this iterator + itr.destroy(); + return evictions; + } // returns true if nvmcache is enabled and we should write this item to // nvmcache. @@ -2064,6 +2138,9 @@ class CacheAllocator : public CacheBase { // free memory monitor std::unique_ptr memMonitor_; + + // background evictor + std::unique_ptr> backgroundEvictor_; // check whether a pool is a slabs pool std::array isCompactCachePool_{}; @@ -2119,6 +2196,7 @@ class CacheAllocator : public CacheBase { // Make this friend to give access to acquire and release friend ReadHandle; friend ReaperAPIWrapper; + friend BackgroundEvictorAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index 1d11b3ef14..56e817ddef 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -31,6 +31,7 @@ #include "cachelib/allocator/NvmAdmissionPolicy.h" #include "cachelib/allocator/PoolOptimizeStrategy.h" #include "cachelib/allocator/RebalanceStrategy.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" #include "cachelib/allocator/Util.h" #include "cachelib/common/EventInterface.h" #include "cachelib/common/Throttler.h" @@ -265,6 +266,12 @@ class CacheAllocatorConfig { std::chrono::seconds regularInterval, std::chrono::seconds ccacheInterval, uint32_t ccacheStepSizePercent); + + // Enable the background evictor - scans a tier to look for objects + // to evict to the next tier + CacheAllocatorConfig& enableBackgroundEvictor( + std::shared_ptr backgroundEvictorStrategy, + std::chrono::milliseconds regularInterval); // This enables an optimization for Pool rebalancing and resizing. // The rough idea is to ensure only the least useful items are evicted when @@ -336,6 +343,12 @@ class CacheAllocatorConfig { compactCacheOptimizeInterval.count() > 0) && poolOptimizeStrategy != nullptr; } + + // @return whether background evictor thread is enabled + bool backgroundEvictorEnabled() const noexcept { + return backgroundEvictorInterval.count() > 0 && + backgroundEvictorStrategy != nullptr; + } // @return whether memory monitor is enabled bool memMonitoringEnabled() const noexcept { @@ -426,6 +439,9 @@ class CacheAllocatorConfig { // time interval to sleep between iterators of rebalancing the pools. std::chrono::milliseconds poolRebalanceInterval{std::chrono::seconds{1}}; + + // time interval to sleep between runs of the background evictor + std::chrono::milliseconds backgroundEvictorInterval{std::chrono::milliseconds{1000}}; // Free slabs pro-actively if the ratio of number of freeallocs to // the number of allocs per slab in a slab class is above this @@ -437,6 +453,9 @@ class CacheAllocatorConfig { // rebalance to avoid alloc fialures. std::shared_ptr defaultPoolRebalanceStrategy{ new RebalanceStrategy{}}; + + // rebalance to avoid alloc fialures. + std::shared_ptr backgroundEvictorStrategy; // time interval to sleep between iterations of pool size optimization, // for regular pools and compact caches @@ -578,6 +597,12 @@ class CacheAllocatorConfig { // skip promote children items in chained when parent fail to promote bool skipPromoteChildrenWhenParentFailed{false}; + // wakeupBg evictor each time there is no memory to allocate new item in topmost tier + bool wakeupBgEvictor {false}; + + // every time there is no space to allocate for particual cid,pid pass this information to BG worker + bool scheduleEviction {false}; + friend CacheT; private: @@ -963,6 +988,15 @@ CacheAllocatorConfig& CacheAllocatorConfig::enablePoolRebalancing( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundEvictor( + std::shared_ptr strategy, + std::chrono::milliseconds interval) { + backgroundEvictorStrategy = strategy; + backgroundEvictorInterval = interval; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enablePoolResizing( std::shared_ptr resizeStrategy, diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 146de6bea7..0e48d55aa9 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -285,6 +285,24 @@ struct ReaperStats { uint64_t avgTraversalTimeMs{0}; }; +// Eviction Stats +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t numEvictedItems{0}; + + // the number of items this worker evicted for pools/classes requested by schedule call + uint64_t numEvictedItemsFromSchedule{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t numTraversals{0}; + + // total number of classes + uint64_t numClasses{0}; + + // eviction size + uint64_t evictionSize{0}; +}; + // CacheMetadata type to export struct CacheMetadata { // allocator_version @@ -305,6 +323,9 @@ struct Stats; // Stats that apply globally in cache and // the ones that are aggregated over all pools struct GlobalCacheStats { + // background eviction stats + BackgroundEvictionStats evictionStats; + // number of calls to CacheAllocator::find uint64_t numCacheGets{0}; diff --git a/cachelib/allocator/FreeThresholdStrategy.cpp b/cachelib/allocator/FreeThresholdStrategy.cpp new file mode 100644 index 0000000000..acafc9d05b --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/FreeThresholdStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + + + +FreeThresholdStrategy::FreeThresholdStrategy(double freeThreshold) + : freeThreshold_(freeThreshold) {} + +size_t FreeThresholdStrategy::calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid ) { + const auto& mpStats = cache.getPoolByTid(pid,tid).getStats().acStats.at(cid); + size_t targetMem = (freeThreshold_ * mpStats.getTotalMemory()) - mpStats.getTotalFreeMemory(); + return std::max(0UL, targetMem / mpStats.allocSize); +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/FreeThresholdStrategy.h b/cachelib/allocator/FreeThresholdStrategy.h new file mode 100644 index 0000000000..853d72f25b --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + + +// Base class for background eviction strategy. +class FreeThresholdStrategy : public BackgroundEvictorStrategy { + +public: + FreeThresholdStrategy(double freeThreshold); + ~FreeThresholdStrategy() {} + + size_t calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid ); +private: + double freeThreshold_; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/KeepFreeStrategy.cpp b/cachelib/allocator/KeepFreeStrategy.cpp new file mode 100644 index 0000000000..fe291b45f9 --- /dev/null +++ b/cachelib/allocator/KeepFreeStrategy.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/KeepFreeStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + + +KeepFreeStrategy::KeepFreeStrategy(size_t nKeepFree) + : nKeepFree_(nKeepFree) {} + +size_t KeepFreeStrategy::calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid ) { + const auto& mpStats = cache.getPoolByTid(pid,tid).getStats().acStats.at(cid); + return std::max(0UL, nKeepFree_ - (mpStats.getTotalFreeMemory() / mpStats.allocSize)); +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/KeepFreeStrategy.h b/cachelib/allocator/KeepFreeStrategy.h new file mode 100644 index 0000000000..bf3ec9a364 --- /dev/null +++ b/cachelib/allocator/KeepFreeStrategy.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" + +namespace facebook { +namespace cachelib { + +class KeepFreeStrategy : public BackgroundEvictorStrategy { +public: + KeepFreeStrategy(size_t nKeepFree); + ~KeepFreeStrategy() {} + + size_t calculateBatchSize(const CacheBase& cache, + unsigned int tid, + PoolId pid, + ClassId cid ); +private: + size_t nKeepFree_; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/memory/MemoryAllocatorStats.h b/cachelib/allocator/memory/MemoryAllocatorStats.h index 947deec664..8d74dc8fe1 100644 --- a/cachelib/allocator/memory/MemoryAllocatorStats.h +++ b/cachelib/allocator/memory/MemoryAllocatorStats.h @@ -54,6 +54,10 @@ struct ACStats { constexpr size_t getTotalFreeMemory() const noexcept { return Slab::kSize * freeSlabs + freeAllocs * allocSize; } + + constexpr size_t getTotalMemory() const noexcept { + return activeAllocs * allocSize; + } }; // structure to query stats corresponding to a MemoryPool diff --git a/cachelib/allocator/tests/CacheBaseTest.cpp b/cachelib/allocator/tests/CacheBaseTest.cpp index 7818034173..3a6b082084 100644 --- a/cachelib/allocator/tests/CacheBaseTest.cpp +++ b/cachelib/allocator/tests/CacheBaseTest.cpp @@ -32,6 +32,8 @@ class CacheBaseTest : public CacheBase, public SlabAllocatorTestBase { memoryPool_(0, 1024, *slabAllocator_, {64}) {} const std::string getCacheName() const override { return cacheName; } const MemoryPool& getPool(PoolId) const override { return memoryPool_; } + //TODO: do we support tiers in CacheBaseTEst + const MemoryPool& getPoolByTid(PoolId, TierId tid) const override { return memoryPool_; } PoolStats getPoolStats(PoolId) const override { return PoolStats(); } AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId) const override { return AllSlabReleaseEvents{}; diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index 34f65e1b15..cff0b39b11 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -59,6 +59,15 @@ Cache::Cache(const CacheConfig& config, allocatorConfig_.enablePoolRebalancing( config_.getRebalanceStrategy(), std::chrono::seconds(config_.poolRebalanceIntervalSec)); + + //for another day + //allocatorConfig_.enablePoolOptimizer( + // config_.getPoolOptimizerStrategy(), + // std::chrono::seconds(config_.poolOptimizerIntervalSec)); + + allocatorConfig_.enableBackgroundEvictor( + config_.getBackgroundEvictorStrategy(), + std::chrono::milliseconds(config_.backgroundEvictorIntervalMilSec)); if (config_.moveOnSlabRelease && movingSync != nullptr) { allocatorConfig_.enableMovingOnSlabRelease( @@ -114,6 +123,9 @@ Cache::Cache(const CacheConfig& config, } }); + allocatorConfig_.wakeupBgEvictor = config_.wakeupBgEvictor; + allocatorConfig_.scheduleEviction = config_.scheduleEviction; + if (config_.enableItemDestructorCheck) { auto removeCB = [&](const typename Allocator::DestructorData& data) { if (!itemRecords_.validate(data)) { @@ -515,6 +527,15 @@ Stats Cache::getStats() const { const auto navyStats = cache_->getNvmCacheStatsMap(); Stats ret; + ret.backgndEvicStats.nEvictedItems = + cacheStats.evictionStats.numEvictedItems; + ret.backgndEvicStats.nEvictedItemsFromSchedule = + cacheStats.evictionStats.numEvictedItemsFromSchedule; + ret.backgndEvicStats.nTraversals = + cacheStats.evictionStats.numTraversals; + ret.backgndEvicStats.evictionSize = + cacheStats.evictionStats.evictionSize; + ret.numEvictions = aggregate.numEvictions(); ret.numItems = aggregate.numItems(); ret.allocAttempts = cacheStats.allocAttempts; @@ -523,6 +544,12 @@ Stats Cache::getStats() const { ret.numCacheGets = cacheStats.numCacheGets; ret.numCacheGetMiss = cacheStats.numCacheGetMiss; ret.numRamDestructorCalls = cacheStats.numRamDestructorCalls; + + ret.numBackgroundEvictions = cacheStats.backgroundEvictorStats.numEvictedItems; + ret.numBackgroundEvictionsFromSchedule = cacheStats.backgroundEvictorStats.numEvictedItemsFromSchedule; + ret.numBackgroundEvictorRuns = cacheStats.backgroundEvictorStats.numTraversals; + ret.numClasses = cacheStats.backgroundEvictorStats.numClasses; + ret.numNvmGets = cacheStats.numNvmGets; ret.numNvmGetMiss = cacheStats.numNvmGetMiss; ret.numNvmGetCoalesced = cacheStats.numNvmGetCoalesced; @@ -565,6 +592,8 @@ Stats Cache::getStats() const { ret.nvmCounters = cache_->getNvmCacheStatsMap(); } + ret.backgroundEvictionClasses = cache_->getBackgroundEvictorClassStats(); + // nvm stats from navy if (!isRamOnly() && !navyStats.empty()) { auto lookup = [&navyStats](const std::string& key) { diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 004f9fe4c7..fd20c64737 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -25,7 +25,27 @@ DECLARE_bool(report_api_latency); namespace facebook { namespace cachelib { namespace cachebench { + +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t nEvictedItems{0}; + + // the number of items this worker evicted for pools/classes requested by schedule call + uint64_t nEvictedItemsFromSchedule{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t nTraversals{0}; + + // number of classes searched + uint64_t nClasses{0}; + + // size of evicted items + uint64_t evictionSize; +}; + struct Stats { + BackgroundEvictionStats backgndEvicStats; + uint64_t numEvictions{0}; uint64_t numItems{0}; @@ -99,6 +119,8 @@ struct Stats { // what to populate since not all of those are interesting when running // cachebench. std::unordered_map nvmCounters; + + std::map backgroundEvictionClasses; // errors from the nvm engine. std::unordered_map nvmErrors; @@ -115,6 +137,17 @@ struct Stats { << std::endl; out << folly::sformat("RAM Evictions : {:,}", numEvictions) << std::endl; + out << folly::sformat("Tier 0 Background Evicted items : {:,}", + backgndEvicStats.nEvictedItems) << std::endl; + out << folly::sformat("Tier 0 Background Evicted items from schedule : {:,}", + backgndEvicStats.nEvictedItemsFromSchedule) << std::endl; + out << folly::sformat("Tier 0 Background Traversals : {:,}", + backgndEvicStats.nTraversals) << std::endl; + out << folly::sformat("Tier 0 Classes searched : {:,}", + backgndEvicStats.nClasses) << std::endl; + out << folly::sformat("Tier 0 Background Evicted Size : {:,}", + backgndEvicStats.evictionSize) << std::endl; + if (numCacheGets > 0) { out << folly::sformat("Cache Gets : {:,}", numCacheGets) << std::endl; out << folly::sformat("Hit Ratio : {:6.2f}%", overallHitRatio) @@ -272,6 +305,13 @@ struct Stats { out << it.first << " : " << it.second << std::endl; } } + + if (!backgroundEvictionClasses.empty()) { + out << "== Class Background Eviction Counters Map ==" << std::endl; + for (const auto& it : backgroundEvictionClasses) { + out << it.first << " : " << it.second << std::endl; + } + } if (numRamDestructorCalls > 0 || numNvmDestructorCalls > 0) { out << folly::sformat("Destructor executed from RAM {}, from NVM {}", diff --git a/cachelib/cachebench/util/CacheConfig.cpp b/cachelib/cachebench/util/CacheConfig.cpp index 2604744bd9..3a44a46188 100644 --- a/cachelib/cachebench/util/CacheConfig.cpp +++ b/cachelib/cachebench/util/CacheConfig.cpp @@ -20,6 +20,9 @@ #include "cachelib/allocator/LruTailAgeStrategy.h" #include "cachelib/allocator/RandomStrategy.h" +#include "cachelib/allocator/KeepFreeStrategy.h" +#include "cachelib/allocator/FreeThresholdStrategy.h" + namespace facebook { namespace cachelib { namespace cachebench { @@ -27,10 +30,15 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, allocator); JSONSetVal(configJson, cacheSizeMB); JSONSetVal(configJson, poolRebalanceIntervalSec); + JSONSetVal(configJson, backgroundEvictorIntervalMilSec); JSONSetVal(configJson, moveOnSlabRelease); JSONSetVal(configJson, rebalanceStrategy); JSONSetVal(configJson, rebalanceMinSlabs); JSONSetVal(configJson, rebalanceDiffRatio); + + JSONSetVal(configJson, backgroundEvictorStrategy); + JSONSetVal(configJson, freeThreshold); + JSONSetVal(configJson, nKeepFree); JSONSetVal(configJson, htBucketPower); JSONSetVal(configJson, htLockPower); @@ -95,6 +103,9 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, persistedCacheDir); JSONSetVal(configJson, usePosixShm); + JSONSetVal(configJson, scheduleEviction); + JSONSetVal(configJson, wakeupBgEvictor); + if (configJson.count("memoryTiers")) { for (auto& it : configJson["memoryTiers"]) { memoryTierConfigs.push_back(MemoryTierConfig(it).getMemoryTierCacheConfig()); @@ -104,7 +115,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { // if you added new fields to the configuration, update the JSONSetVal // to make them available for the json configs and increment the size // below - checkCorrectSize(); + checkCorrectSize(); if (numPools != poolSizes.size()) { throw std::invalid_argument(folly::sformat( @@ -134,6 +145,21 @@ std::shared_ptr CacheConfig::getRebalanceStrategy() const { } } +std::shared_ptr CacheConfig::getBackgroundEvictorStrategy() const { + if (backgroundEvictorIntervalMilSec == 0) { + return nullptr; + } + + if (backgroundEvictorStrategy == "free-threshold") { + return std::make_shared(freeThreshold); + } else if (backgroundEvictorStrategy == "keep-free") { + return std::make_shared(nKeepFree); + } else { + //default! + return std::make_shared(freeThreshold); + } +} + MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, file); diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index f09d5966bd..027ee67456 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -20,6 +20,7 @@ #include "cachelib/allocator/CacheAllocator.h" #include "cachelib/allocator/RebalanceStrategy.h" +#include "cachelib/allocator/BackgroundEvictorStrategy.h" #include "cachelib/cachebench/util/JSONConfig.h" #include "cachelib/common/Ticker.h" #include "cachelib/navy/common/Device.h" @@ -71,7 +72,11 @@ struct CacheConfig : public JSONConfig { uint64_t cacheSizeMB{0}; uint64_t poolRebalanceIntervalSec{0}; + uint64_t backgroundEvictorIntervalMilSec{0}; std::string rebalanceStrategy; + std::string backgroundEvictorStrategy; + double freeThreshold{0.01}; //keep 1% of space free + uint32_t nKeepFree{100}; //keep at most 100 slots free uint64_t rebalanceMinSlabs{1}; double rebalanceDiffRatio{0.25}; bool moveOnSlabRelease{false}; @@ -283,11 +288,15 @@ struct CacheConfig : public JSONConfig { // this verifies whether the feature affects throughputs. bool enableItemDestructor{false}; + bool wakeupBgEvictor {false}; + bool scheduleEviction {false}; + explicit CacheConfig(const folly::dynamic& configJson); CacheConfig() {} std::shared_ptr getRebalanceStrategy() const; + std::shared_ptr getBackgroundEvictorStrategy() const; }; } // namespace cachebench } // namespace cachelib