Background evictor thread

cachelib/allocator/BackgroundEvictor-inl.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) Intel and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+namespace facebook {
+namespace cachelib {
+
+
+template <typename CacheT>
+BackgroundEvictor<CacheT>::BackgroundEvictor(Cache& cache,
+                               std::shared_ptr<BackgroundEvictorStrategy> strategy,
+                               unsigned int tid)
+    : cache_(cache),
+      strategy_(strategy),
+      tid_(tid) {
+}
+
+template <typename CacheT>
+BackgroundEvictor<CacheT>::~BackgroundEvictor() { stop(std::chrono::seconds(0)); }
+
+template <typename CacheT>
+void BackgroundEvictor<CacheT>::work() {
+  try {
+    if (!tasks_.empty()) {
+      while (auto entry = tasks_.try_dequeue()) {
+        auto [pid, cid] = entry.value();
+        auto batch = strategy_->calculateBatchSize(cache_, tid_, pid, cid);
+        stats.evictionSize.add(batch);
+        auto evicted = BackgroundEvictorAPIWrapper<CacheT>::traverseAndEvictItems(cache_,
+                tid_,pid,cid,batch);
+        stats.numEvictedItemsFromSchedule.inc();
+        stats.numTraversals.inc();
+      }
+    } else {
+      for (const auto pid : cache_.getRegularPoolIds()) {
+        //check if the pool is full - probably should be if tier is full
+        if (cache_.getPoolByTid(pid,tid_).allSlabsAllocated()) {
+          checkAndRun(pid);
+        }
+      }
+    }
+  } catch (const std::exception& ex) {
+    XLOGF(ERR, "BackgroundEvictor interrupted due to exception: {}", ex.what());
+  }
+}
+
+// Look for classes that exceed the target memory capacity
+// and return those for eviction
+template <typename CacheT>
+void BackgroundEvictor<CacheT>::checkAndRun(PoolId pid) {    
+  const auto& mpStats = cache_.getPoolByTid(pid,tid_).getStats();
+  unsigned int evictions = 0;
+  unsigned int classes = 0;
+  for (auto& cid : mpStats.classIds) {
+      classes++;
+      auto batch = strategy_->calculateBatchSize(cache_,tid_,pid,cid);
+      if (!batch) {
+        continue;
+      }
+
+      stats.evictionSize.add(batch);
+      //try evicting BATCH items from the class in order to reach free target
+      auto evicted =
+          BackgroundEvictorAPIWrapper<CacheT>::traverseAndEvictItems(cache_,
+              tid_,pid,cid,batch);
+
+      evictions += evicted;
+      const size_t cid_id = (size_t)mpStats.acStats.at(cid).allocSize;
+      auto it = evictions_per_class_.find(cid_id);
+      if (it != evictions_per_class_.end()) {
+          it->second += evicted;
+      } else {
+          evictions_per_class_[cid_id] = 0;
+      }
+  }
+  stats.numEvictedItems.add(evictions);
+  stats.numClasses.add(classes);
+  stats.numTraversals.inc();
+}
+
+template <typename CacheT>
+BackgroundEvictionStats BackgroundEvictor<CacheT>::getStats() const noexcept {
+  BackgroundEvictionStats evicStats;
+  evicStats.numEvictedItems = stats.numEvictedItems.get();
+  evicStats.numEvictedItemsFromSchedule = stats.numEvictedItemsFromSchedule.get();
+  evicStats.numTraversals = stats.numTraversals.get();
+  evicStats.numClasses = stats.numClasses.get();
+  evicStats.evictionSize = stats.evictionSize.get();
+
+  return evicStats;
+}
+
+template <typename CacheT>
+std::map<uint32_t,uint64_t> BackgroundEvictor<CacheT>::getClassStats() const noexcept {
+  return evictions_per_class_;
+}
+
+} // namespace cachelib
+} // namespace facebook

cachelib/allocator/BackgroundEvictor.h

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) Intel and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <gtest/gtest_prod.h>
+#include <folly/concurrency/UnboundedQueue.h>
+
+#include "cachelib/allocator/CacheStats.h"
+#include "cachelib/common/PeriodicWorker.h"
+#include "cachelib/allocator/BackgroundEvictorStrategy.h"
+#include "cachelib/common/AtomicCounter.h"
+
+
+namespace facebook {
+namespace cachelib {
+
+// wrapper that exposes the private APIs of CacheType that are specifically
+// needed for the eviction.
+template <typename C>
+struct BackgroundEvictorAPIWrapper {
+
+  static size_t traverseAndEvictItems(C& cache,
+          unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) {
+    return cache.traverseAndEvictItems(tid,pid,cid,batch);
+  }
+};
+
+struct BackgroundEvictorStats {
+  // items evicted
+  AtomicCounter numEvictedItems{0};
+
+  // items evicted from schedule
+  AtomicCounter numEvictedItemsFromSchedule;
+
+  // traversals
+  AtomicCounter numTraversals{0};
+
+  // total number of classes
+  AtomicCounter numClasses{0};
+
+  // item eviction size
+  AtomicCounter evictionSize{0};
+};
+
+// Periodic worker that evicts items from tiers in batches
+// The primary aim is to reduce insertion times for new items in the
+// cache
+template <typename CacheT>
+class BackgroundEvictor : public PeriodicWorker {
+ public:
+  using Cache = CacheT;
+  // @param cache               the cache interface
+  // @param target_free         the target amount of memory to keep free in 
+  //                            this tier
+  // @param tier id             memory tier to perform eviction on 
+  BackgroundEvictor(Cache& cache,
+                    std::shared_ptr<BackgroundEvictorStrategy> strategy,
+                    unsigned int tid);
+
+  ~BackgroundEvictor() override;
+
+  void schedule(size_t pid, size_t cid) {
+      tasks_.enqueue(std::make_pair(pid,cid));
+  }
+
+  BackgroundEvictionStats getStats() const noexcept;
+
+  std::map<uint32_t,uint64_t> getClassStats() const noexcept;
+
+ private:
+  // cache allocator's interface for evicting
+
+  using Item = typename Cache::Item;
+
+  Cache& cache_;
+  std::shared_ptr<BackgroundEvictorStrategy> strategy_;
+  unsigned int tid_;
+  folly::UMPSCQueue<std::pair<size_t,size_t>,true> tasks_;
+
+  // implements the actual logic of running the background evictor
+  void work() override final;
+  void checkAndRun(PoolId pid);
+
+  BackgroundEvictorStats stats;
+
+  std::map<uint32_t,uint64_t> evictions_per_class_;
+};
+} // namespace cachelib
+} // namespace facebook
+
+#include "cachelib/allocator/BackgroundEvictor-inl.h"

cachelib/allocator/BackgroundEvictorStrategy.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "cachelib/allocator/Cache.h"
+
+namespace facebook {
+namespace cachelib {
+
+
+// Base class for background eviction strategy.
+class BackgroundEvictorStrategy {
+
+public:
+  virtual size_t calculateBatchSize(const CacheBase& cache,
+                                       unsigned int tid,
+                                       PoolId pid,
+                                       ClassId cid ) = 0;
+};
+
+} // namespace cachelib
+} // namespace facebook

cachelib/allocator/CMakeLists.txt

@@ -35,8 +35,10 @@ add_library (cachelib_allocator
     CCacheManager.cpp
     ContainerTypes.cpp
     FreeMemStrategy.cpp
+    FreeThresholdStrategy.cpp
     HitsPerSlabStrategy.cpp
     LruTailAgeStrategy.cpp
+    KeepFreeStrategy.cpp
     MarginalHitsOptimizeStrategy.cpp
     MarginalHitsStrategy.cpp
     memory/AllocationClass.cpp

cachelib/allocator/Cache.h

@@ -93,6 +93,12 @@ class CacheBase {
   //
   // @param poolId    The pool id to query
   virtual const MemoryPool& getPool(PoolId poolId) const = 0;
+
+  // Get the reference  to a memory pool using a tier id, for stats purposes
+  //
+  // @param poolId    The pool id to query
+  // @param tierId    The tier of the pool id
+  virtual const MemoryPool& getPoolByTid(PoolId poolId, TierId tid) const = 0;
 
   // Get Pool specific stats (regular pools). This includes stats from the
   // Memory Pool and also the cache.

cachelib/allocator/CacheAllocator-inl.h

@@ -325,6 +325,12 @@ void CacheAllocator<CacheTrait>::initWorkers() {
                           config_.poolOptimizeStrategy,
                           config_.ccacheOptimizeStepSizePercent);
   }
+
+  if (config_.backgroundEvictorEnabled()) {
+      startNewBackgroundEvictor(config_.backgroundEvictorInterval,
+                                config_.backgroundEvictorStrategy,
+                                0); //right now default to tier 0);
+  }
 }
 
 template <typename CacheTrait>
@@ -376,6 +382,14 @@ CacheAllocator<CacheTrait>::allocateInternalTier(TierId tid,
   //       Should we support eviction between memory tiers (e.g. from DRAM to PMEM)?
   if (memory == nullptr && !config_.disableEviction) {
     memory = findEviction(tid, pid, cid);
+
+    if (backgroundEvictor_ && config_.scheduleEviction) {
+      backgroundEvictor_->schedule(pid,cid);
+    }
+
+    if (backgroundEvictor_ && config_.wakeupBgEvictor) {
+      backgroundEvictor_->wakeUp();
+    }
   }
 
   ItemHandle handle;
@@ -3381,6 +3395,7 @@ bool CacheAllocator<CacheTrait>::stopWorkers(std::chrono::seconds timeout) {
   success &= stopPoolResizer(timeout);
   success &= stopMemMonitor(timeout);
   success &= stopReaper(timeout);
+  success &= stopBackgroundEvictor(timeout);
   return success;
 }
 
@@ -3661,6 +3676,7 @@ GlobalCacheStats CacheAllocator<CacheTrait>::getGlobalCacheStats() const {
   ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false;
   ret.nvmUpTime = currTime - getNVMCacheCreationTime();
   ret.reaperStats = getReaperStats();
+  ret.evictionStats = getBackgroundEvictorStats();
   ret.numActiveHandles = getNumActiveHandles();
 
   return ret;
@@ -3759,6 +3775,7 @@ bool CacheAllocator<CacheTrait>::startNewPoolRebalancer(
                         freeAllocThreshold);
 }
 
+
 template <typename CacheTrait>
 bool CacheAllocator<CacheTrait>::startNewPoolResizer(
     std::chrono::milliseconds interval,
@@ -3796,6 +3813,14 @@ bool CacheAllocator<CacheTrait>::startNewReaper(
   return startNewWorker("Reaper", reaper_, interval, reaperThrottleConfig);
 }
 
+template <typename CacheTrait>
+bool CacheAllocator<CacheTrait>::startNewBackgroundEvictor(
+    std::chrono::milliseconds interval,
+    std::shared_ptr<BackgroundEvictorStrategy> strategy,
+    unsigned int tid ) {
+  return startNewWorker("BackgroundEvictor", backgroundEvictor_, interval, strategy, tid);
+}
+
 template <typename CacheTrait>
 bool CacheAllocator<CacheTrait>::stopPoolRebalancer(
     std::chrono::seconds timeout) {
@@ -3823,6 +3848,12 @@ bool CacheAllocator<CacheTrait>::stopReaper(std::chrono::seconds timeout) {
   return stopWorker("Reaper", reaper_, timeout);
 }
 
+template <typename CacheTrait>
+bool CacheAllocator<CacheTrait>::stopBackgroundEvictor(
+    std::chrono::seconds timeout) {
+  return stopWorker("BackgroundEvictor", backgroundEvictor_, timeout);
+}
+
 template <typename CacheTrait>
 bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
   const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector<CacheMemoryTierConfig>& config */) {