Add counter to track global evictions across RAM and NVM cache

Summary:
When HybridCache is enabled, we don't have a global eviction counter (i.e. when an item is leaving the cache entirely due to eviction).
- `ram.evictions`: item evicted from ram cache but it can be inserted into nvm cache
- `nvm.evictions`: item evicted from nvm cache but it can be still in ram cache
- `ram.destructors`: item leaves the cache entirely either because of removal or eviction from ram; must enable itemDestructor
- `nvm.destructors` : item leaves the cache entirely either because of removal or eviction from nvm; must enable itemDestructor

In this diff, we add `cache.evictions`: item leaves the cache entirely because of evictions from ram or nvm. No need to enable itemDestructor.

Reviewed By: therealgymmy, jaesoo-fb

Differential Revision: D40909473

fbshipit-source-id: 4d61d1434f073c95ea59da3abfd4152b17b0f67c

Author: Jiayue Bao

cachelib/allocator/CacheAllocator-inl.h

@@ -761,17 +761,27 @@ CacheAllocator<CacheTrait>::releaseBackToAllocator(Item& it,
 
   // only skip destructor for evicted items that are either in the queue to put
   // into nvm or already in nvm
-  if (!nascent && config_.itemDestructor &&
-      (ctx != RemoveContext::kEviction || !it.isNvmClean() ||
-       it.isNvmEvicted())) {
-    try {
-      config_.itemDestructor(DestructorData{
-          ctx, it, viewAsChainedAllocsRange(it), allocInfo.poolId});
-      stats().numRamDestructorCalls.inc();
-    } catch (const std::exception& e) {
-      stats().numDestructorExceptions.inc();
-      XLOG_EVERY_N(INFO, 100)
-          << "Catch exception from user's item destructor: " << e.what();
+  bool skipDestructor =
+      nascent || (ctx == RemoveContext::kEviction &&
+                  // When this item is queued for NvmCache, it will be marked
+                  // as clean and the NvmEvicted bit will also be set to false.
+                  // Refer to NvmCache::put()
+                  it.isNvmClean() && !it.isNvmEvicted());
+  if (!skipDestructor) {
+    if (ctx == RemoveContext::kEviction) {
+      stats().numCacheEvictions.inc();
+    }
+    // execute ItemDestructor
+    if (config_.itemDestructor) {
+      try {
+        config_.itemDestructor(DestructorData{
+            ctx, it, viewAsChainedAllocsRange(it), allocInfo.poolId});
+        stats().numRamDestructorCalls.inc();
+      } catch (const std::exception& e) {
+        stats().numDestructorExceptions.inc();
+        XLOG_EVERY_N(INFO, 100)
+            << "Catch exception from user's item destructor: " << e.what();
+      }
     }
   }
 

cachelib/allocator/CacheStats.cpp

@@ -51,14 +51,15 @@ struct SizeVerify {};
 
 void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const {
 #ifndef SKIP_SIZE_VERIFY
-  SizeVerify<sizeof(Stats)> a = SizeVerify<16160>{};
+  SizeVerify<sizeof(Stats)> a = SizeVerify<16176>{};
   std::ignore = a;
 #endif
   ret.numCacheGets = numCacheGets.get();
   ret.numCacheGetMiss = numCacheGetMiss.get();
   ret.numCacheGetExpiries = numCacheGetExpiries.get();
   ret.numCacheRemoves = numCacheRemoves.get();
   ret.numCacheRemoveRamHits = numCacheRemoveRamHits.get();
+  ret.numCacheEvictions = numCacheEvictions.get();
   ret.numRamDestructorCalls = numRamDestructorCalls.get();
   ret.numDestructorExceptions = numDestructorExceptions.get();
 

cachelib/allocator/CacheStats.h

@@ -377,6 +377,9 @@ struct GlobalCacheStats {
   // number of evictions from NvmCache
   uint64_t numNvmEvictions{0};
 
+  // number of evictions where items leave both RAM and NvmCache entirely
+  uint64_t numCacheEvictions{0};
+
   // number of evictions from nvm that found an inconsistent state in RAM
   uint64_t numNvmUncleanEvict{0};
 

cachelib/allocator/CacheStatsInternal.h

@@ -51,6 +51,9 @@ struct Stats {
   // number of remove calls that resulted in a ram hit
   TLCounter numCacheRemoveRamHits{0};
 
+  // number of evictions where items leave both RAM and NvmCache entirely
+  AtomicCounter numCacheEvictions{0};
+
   // number of item destructor calls from ram
   TLCounter numRamDestructorCalls{0};
 

cachelib/allocator/nvmcache/NvmCache-inl.h

@@ -381,8 +381,15 @@ void NvmCache<C>::evictCB(HashedKey hk,
     }
   }
 
+  if (!needDestructor) {
+    return;
+  }
+
+  if (event != cachelib::navy::DestructorEvent::Removed) {
+    stats().numCacheEvictions.inc();
+  }
   // ItemDestructor
-  if (itemDestructor_ && needDestructor) {
+  if (itemDestructor_) {
     // create the item on heap instead of memory pool to avoid allocation
     // failure and evictions from cache for a temporary item.
     auto iobuf = createItemAsIOBuf(hk.key(), nvmItem);

cachelib/cachebench/cache/Cache-inl.h

@@ -609,6 +609,7 @@ Stats Cache<Allocator>::getStats() const {
 
   ret.numCacheGets = cacheStats.numCacheGets;
   ret.numCacheGetMiss = cacheStats.numCacheGetMiss;
+  ret.numCacheEvictions = cacheStats.numCacheEvictions;
   ret.numRamDestructorCalls = cacheStats.numRamDestructorCalls;
   ret.numNvmGets = cacheStats.numNvmGets;
   ret.numNvmGetMiss = cacheStats.numNvmGetMiss;

cachelib/cachebench/cache/CacheStats.h

@@ -37,6 +37,7 @@ struct Stats {
 
   uint64_t numCacheGets{0};
   uint64_t numCacheGetMiss{0};
+  uint64_t numCacheEvictions{0};
   uint64_t numRamDestructorCalls{0};
   uint64_t numNvmGets{0};
   uint64_t numNvmGetMiss{0};
@@ -293,6 +294,11 @@ struct Stats {
                             numRamDestructorCalls, numNvmDestructorCalls)
           << std::endl;
     }
+
+    if (numCacheEvictions > 0) {
+      out << folly::sformat("Total eviction executed {}", numCacheEvictions)
+          << std::endl;
+    }
   }
 
   uint64_t getTotalMisses() const {