diff --git a/cachelib/allocator/DynamicFreeThresholdStrategy.cpp b/cachelib/allocator/DynamicFreeThresholdStrategy.cpp index 6a70fed2ef..a19ca8c63d 100644 --- a/cachelib/allocator/DynamicFreeThresholdStrategy.cpp +++ b/cachelib/allocator/DynamicFreeThresholdStrategy.cpp @@ -14,120 +14,144 @@ * limitations under the License. */ - //class-specific dynamic free (high) threshold strategy - +// class-specific dynamic free (high) threshold strategy #include "cachelib/allocator/DynamicFreeThresholdStrategy.h" -#include "cachelib/allocator/memory/MemoryPoolManager.h" -#include "cachelib/allocator/memory/MemoryAllocator.h" -#include "cachelib/allocator/Cache.h" -#include "cachelib/allocator/CacheStats.h" + #include +#include + +#include #include -#include -#include -#include #include + +#include "cachelib/allocator/Cache.h" +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/allocator/memory/MemoryAllocator.h" +#include "cachelib/allocator/memory/MemoryPoolManager.h" namespace facebook { namespace cachelib { - - - -DynamicFreeThresholdStrategy::DynamicFreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t maxEvictionBatch, uint64_t minEvictionBatch, double highEvictionDelt ) - : lowEvictionAcWatermark(lowEvictionAcWatermark), highEvictionAcWatermark(highEvictionAcWatermark), maxEvictionBatch(maxEvictionBatch), minEvictionBatch(minEvictionBatch), highEvictionDelta(highEvictionDelt), - highEvictionAcWatermarks(CacheBase::kMaxTiers, - std::vector>>(MemoryPoolManager::kMaxPools, - std::vector>(MemoryAllocator::kMaxClasses, - std::vector(3, highEvictionAcWatermark)))), //initialize highEvictionAcWatermarks - /*acBenefits(CacheBase::kMaxTiers, - std::vector>>(MemoryPoolManager::kMaxPools, - std::vector>(MemoryAllocator::kMaxClasses, - std::vector(2, 0.0)))), //initialize acBenefits - */ - acLatencies(CacheBase::kMaxTiers, - std::vector>>(MemoryPoolManager::kMaxPools, - std::vector>(MemoryAllocator::kMaxClasses, - std::vector(2, 0.0)))), - acToFreeMemPercents(CacheBase::kMaxTiers, - std::vector>>(MemoryPoolManager::kMaxPools, - std::vector>(MemoryAllocator::kMaxClasses, - std::vector(2, 0.0)))) {} //initialize acToFreeMemPercents - -std::vector DynamicFreeThresholdStrategy::calculateBatchSizes -(const CacheBase& cache, std::vector acVec) -{ - - std::vector batches{}; //contain number of items to evict for ac classes in the batch - - +DynamicFreeThresholdStrategy::DynamicFreeThresholdStrategy( + double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch, + double highEvictionDelt) + : lowEvictionAcWatermark(lowEvictionAcWatermark), + highEvictionAcWatermark(highEvictionAcWatermark), + maxEvictionBatch(maxEvictionBatch), + minEvictionBatch(minEvictionBatch), + highEvictionDelta(highEvictionDelt), + highEvictionAcWatermarks( + CacheBase::kMaxTiers, + std::vector>>( + MemoryPoolManager::kMaxPools, + std::vector>( + MemoryAllocator::kMaxClasses, + std::vector( + 3, + highEvictionAcWatermark)))), // initialize + // highEvictionAcWatermarks + + acLatencies( + CacheBase::kMaxTiers, + std::vector>>( + MemoryPoolManager::kMaxPools, + std::vector>(MemoryAllocator::kMaxClasses, + std::vector(2, 0.0)))), + acToFreeMemPercents( + CacheBase::kMaxTiers, + std::vector>>( + MemoryPoolManager::kMaxPools, + std::vector>(MemoryAllocator::kMaxClasses, + std::vector(2, 0.0)))) { +} // initialize acToFreeMemPercents + +std::vector DynamicFreeThresholdStrategy::calculateBatchSizes( + const CacheBase& cache, std::vector acVec) { + std::vector batches{}; // contain number of items to evict for ac + // classes in the batch for (auto [tid, pid, cid] : acVec) { - - auto stats = cache.getAllocationClassStats(tid, pid, cid); //ac class stats - auto acFree = stats.approxFreePercent; //amount of free memory in the ac class - - if (acFree >= lowEvictionAcWatermark) { //if the amount of free memory in the ac class is above lowEvictionAcWatermark, - batches.push_back(0); //we do not evict + auto stats = cache.getAllocationClassStats(tid, pid, cid); // ac class stats + auto acFree = + stats.approxFreePercent; // amount of free memory in the ac class + + if (acFree >= lowEvictionAcWatermark) { // if the amount of free memory in + // the ac class is above + // lowEvictionAcWatermark, + batches.push_back(0); // we do not evict } else { - auto acHighThresholdAtI = highEvictionAcWatermarks[tid][pid][cid][0]; //current high threshold - auto acHighThresholdAtINew = highEvictionAcWatermarks[tid][pid][cid][0]; //new high threshold, will be adjusted - auto acHighThresholdAtIMinus1 = highEvictionAcWatermarks[tid][pid][cid][1]; //previous high threshold - auto acHighThresholdAtIMinus2 = highEvictionAcWatermarks[tid][pid][cid][2]; //previous of previous high threshold + auto acHighThresholdAtI = + highEvictionAcWatermarks[tid][pid][cid][0]; // current high threshold + auto acHighThresholdAtINew = + highEvictionAcWatermarks[tid][pid][cid][0]; // new high threshold, + // will be adjusted + auto acHighThresholdAtIMinus1 = + highEvictionAcWatermarks[tid][pid][cid][1]; // previous high threshold + auto acHighThresholdAtIMinus2 = + highEvictionAcWatermarks[tid][pid][cid][2]; // previous of previous + // high threshold auto toFreeMemPercentAtI = acToFreeMemPercents[tid][pid][cid][0]; - auto toFreeMemPercentAtIMinus1 = acToFreeMemPercents[tid][pid][cid][1]; //previous amount of memory to free up in the ac class - auto acAllocLatencyNs = cache.getAllocationClassStats(tid, pid, cid).allocLatencyNs.estimate(); //moving avg latency estimation for ac class - + auto toFreeMemPercentAtIMinus1 = + acToFreeMemPercents[tid][pid][cid][1]; // previous amount of memory to + // free up in the ac class + auto acAllocLatencyNs = + cache.getAllocationClassStats(tid, pid, cid) + .allocLatencyNs.estimate(); // moving avg latency estimation for + // ac class + calculateLatency(acAllocLatencyNs, tid, pid, cid); - std::default_random_engine generator; std::normal_distribution distribution(1); - double number=distribution(generator); - acHighThresholdAtINew+= number*highEvictionDelta; - std::normal_distribution randDistribution(0.0,1.0); - double randoNum=randDistribution(generator); - - if (acLatencies[tid][pid][cid][0] < acLatencies[tid][pid][cid][1]){ //if current latency reduced - // then increase threshold to increase eviction - //acHighThresholdAtINew+= number*highEvictionDelta; - highEvictionAcWatermarks[tid][pid][cid][0]=acHighThresholdAtINew; - highEvictionAcWatermarks[tid][pid][cid][1] = acHighThresholdAtI; + double number = distribution(generator); + acHighThresholdAtINew += number * highEvictionDelta; + std::normal_distribution randDistribution(0.0, 1.0); + double randoNum = randDistribution(generator); + + if (acLatencies[tid][pid][cid][0] < + acLatencies[tid][pid][cid][1]) { // if current latency reduced + // then increase threshold to increase eviction + // acHighThresholdAtINew+= number*highEvictionDelta; + highEvictionAcWatermarks[tid][pid][cid][0] = acHighThresholdAtINew; + highEvictionAcWatermarks[tid][pid][cid][1] = acHighThresholdAtI; highEvictionAcWatermarks[tid][pid][cid][2] = acHighThresholdAtIMinus1; - } - //rand boundary, - float diff = acLatencies[tid][pid][cid][0] - acLatencies[tid][pid][cid][1]; - //calculate metropolis acceptance criterion - auto t = initial_latency / ( (float) tid + 1); + } + // rand boundary, + float diff = + acLatencies[tid][pid][cid][0] - acLatencies[tid][pid][cid][1]; + // calculate metropolis acceptance criterion + auto t = initial_latency / ((float)tid + 1); float metropolis = exp(-abs(diff) / t); - if (diff < 0 || randoNum < metropolis){ - //acHighThresholdAtINew-= number*highEvictionDelta; - highEvictionAcWatermarks[tid][pid][cid][0]=acHighThresholdAtINew; - highEvictionAcWatermarks[tid][pid][cid][1] = acHighThresholdAtI; + if (diff < 0 || randoNum < metropolis) { + // acHighThresholdAtINew-= number*highEvictionDelta; + highEvictionAcWatermarks[tid][pid][cid][0] = acHighThresholdAtINew; + highEvictionAcWatermarks[tid][pid][cid][1] = acHighThresholdAtI; highEvictionAcWatermarks[tid][pid][cid][2] = acHighThresholdAtIMinus1; - } - - acHighThresholdAtINew = std::max(acHighThresholdAtINew, lowEvictionAcWatermark); // high threshold cannot be less than the low threshold - auto CurrentItemsPercent=100-acFree; - auto toFreeMemPercent=CurrentItemsPercent-acHighThresholdAtINew; - - - acToFreeMemPercents[tid][pid][cid][1] = toFreeMemPercentAtI; //update acToFreeMemPercents - acToFreeMemPercents[tid][pid][cid][0] = toFreeMemPercent; //update acToFreeMemPercents - - - - - - auto toFreeItems = static_cast(toFreeMemPercent * stats.memorySize / stats.allocSize); //calculate number of items to evict for current ac class - batches.push_back(toFreeItems); //append batches - - - - } - + } + + acHighThresholdAtINew = + std::max(acHighThresholdAtINew, + lowEvictionAcWatermark); // high threshold cannot be less + // than the low threshold + auto CurrentItemsPercent = 100 - acFree; + auto toFreeMemPercent = CurrentItemsPercent - acHighThresholdAtINew; + + acToFreeMemPercents[tid][pid][cid][1] = + toFreeMemPercentAtI; // update acToFreeMemPercents + acToFreeMemPercents[tid][pid][cid][0] = + toFreeMemPercent; // update acToFreeMemPercents + + auto toFreeItems = + static_cast(toFreeMemPercent * stats.memorySize / + stats.allocSize); // calculate number of items to + // evict for current ac class + batches.push_back(toFreeItems); // append batches } - + } + if (batches.size() == 0) { return batches; } @@ -135,29 +159,28 @@ std::vector DynamicFreeThresholdStrategy::calculateBatchSizes if (maxBatch == 0) return batches; - std::transform(batches.begin(), batches.end(), batches.begin(), [&](auto numItems){ - if (numItems == 0) { - return 0UL; - } + std::transform( + batches.begin(), batches.end(), batches.begin(), [&](auto numItems) { + if (numItems == 0) { + return 0UL; + } - auto cappedBatchSize = maxEvictionBatch * numItems / maxBatch; - if (cappedBatchSize < minEvictionBatch) - return minEvictionBatch; - else - return cappedBatchSize; - }); + auto cappedBatchSize = maxEvictionBatch * numItems / maxBatch; + if (cappedBatchSize < minEvictionBatch) + return minEvictionBatch; + else + return cappedBatchSize; + }); return batches; } - - void DynamicFreeThresholdStrategy::calculateLatency(uint64_t acLatency, unsigned int tid, PoolId pid, ClassId cid){ - - auto best_latency= acLatencies[tid][pid][cid][0]; - acLatencies[tid][pid][cid][1]=best_latency; - acLatencies[tid][pid][cid][0]=acLatency; - - - +void DynamicFreeThresholdStrategy::calculateLatency(uint64_t acLatency, + unsigned int tid, + PoolId pid, + ClassId cid) { + auto best_latency = acLatencies[tid][pid][cid][0]; + acLatencies[tid][pid][cid][1] = best_latency; + acLatencies[tid][pid][cid][0] = acLatency; } } // namespace cachelib } // namespace facebook diff --git a/cachelib/allocator/DynamicFreeThresholdStrategy.h b/cachelib/allocator/DynamicFreeThresholdStrategy.h index cda6408121..98e2a78f0e 100644 --- a/cachelib/allocator/DynamicFreeThresholdStrategy.h +++ b/cachelib/allocator/DynamicFreeThresholdStrategy.h @@ -16,57 +16,79 @@ #pragma once +#include + #include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheStats.h" -#include namespace facebook { namespace cachelib { - // Base class for background eviction strategy. class DynamicFreeThresholdStrategy : public BackgroundMoverStrategy { - -public: - DynamicFreeThresholdStrategy(double lowEvictionAcWatermark, double highEvictionAcWatermark, uint64_t maxEvictionBatch, uint64_t minEvictionBatch, double highEvictionDelta); + public: + DynamicFreeThresholdStrategy(double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch, + double highEvictionDelta); ~DynamicFreeThresholdStrategy() {} - std::vector calculateBatchSizes(const CacheBase& cache, std::vector acVecs); //function to calculate number of items to evict for alloc. classes in the batch - - + std::vector calculateBatchSizes( + const CacheBase& cache, + std::vector acVecs); // function to calculate number + // of items to evict for alloc. + // classes in the batch -private: - double lowEvictionAcWatermark{2.0}; //threshold to activate eviction in a certain alloc. class - //for now: static threshold, same for all alloc. classes, TODO: implement dynamic low threshold - double highEvictionAcWatermark{5.0}; //threshold to calculate number of items to evict from a certain alloc. class - //this threshold is adjusted internally, individually for each ac class + private: + double lowEvictionAcWatermark{2.0}; // threshold to activate eviction in a + // certain alloc. class for now: static + // threshold, same for all alloc. classes, + // TODO: implement dynamic low threshold + double highEvictionAcWatermark{5.0}; // threshold to calculate number of items + // to evict from a certain alloc. class + // this threshold is adjusted internally, + // individually for each ac class uint64_t maxEvictionBatch{40}; - uint64_t minEvictionBatch{50}; - double highEvictionDelta{0.3}; //step size of hill-climbing algorithm - //TODO: tune this param, use access freq or other access stat as basis, perhaps use the benefit function to adjust this param (binned)? + uint64_t minEvictionBatch{5}; + double highEvictionDelta{0.3}; // step size of hill-climbing algorithm + // TODO: tune this param, use access freq or + // other access stat as basis, perhaps use the + // benefit function to adjust this param + // (binned)? double acLatencyDelta{0.1}; - double initial_latency {10.0}; - double alpha {0.99}; - std::vector>>> highEvictionAcWatermarks; //individual dynamic thresholds for each ac class - //index 0 for i-th window - //index 1 for i-1 window - //index 2 for i-2 window + double initial_latency{10.0}; + double alpha{0.99}; + std::vector>>> + highEvictionAcWatermarks; // individual dynamic thresholds for each ac + // class + // index 0 for i-th window + // index 1 for i-1 window + // index 2 for i-2 window + + std::vector>>> + acLatencies; // benefit value for each ac class, for now: derived from + // moving avg alloc. latency + + // index 0 for current benefit (i-th window) + // index 1 for previous benefit (i-1 window) - //std::vector>>> acBenefits; //benefit value for each ac class, for now: derived from moving avg alloc. latency - std::vector>>> acLatencies; //benefit value for each ac class, for now: derived from moving avg alloc. latency - - //index 0 for current benefit (i-th window) - //index 1 for previous benefit (i-1 window) - - std::vector>>> acToFreeMemPercents; //amount of memory to free up for each ac class (logging it for comparison purposes) + std::vector>>> + acToFreeMemPercents; // amount of memory to free up for each ac class + // (logging it for comparison purposes) - //index 0 for current toFreeMemPercent (i-th window) - //index 1 for previous toFreeMemPercent (i-1 window) + // index 0 for current toFreeMemPercent (i-th window) + // index 1 for previous toFreeMemPercent (i-1 window) -private: - //void calculateBenefitMig(uint64_t p99, unsigned int tid, PoolId pid, ClassId cid); //function to calculate the benefit of eviction for a certain ac class - void calculateLatency(uint64_t p99, unsigned int tid, PoolId pid, ClassId cid); + private: + // void calculateBenefitMig(uint64_t p99, unsigned int tid, PoolId pid, + // ClassId cid); //function to calculate the benefit of eviction for a certain + // ac class + void calculateLatency(uint64_t p99, + unsigned int tid, + PoolId pid, + ClassId cid); }; } // namespace cachelib diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index 28698ac138..a0ec622eae 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -27,6 +27,7 @@ TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileInvalid) { this->testMulti TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileValid) { this->testMultiTiersFromFileValid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(true); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsSysVValid) { this->testMultiTiersNumaBindingsSysVValid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsPosixValid) { this->testMultiTiersNumaBindingsPosixValid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index 49e1444950..ad62fe2346 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -16,14 +16,14 @@ #pragma once +#include + #include "cachelib/allocator/CacheAllocatorConfig.h" -#include "cachelib/allocator/MemoryTierCacheConfig.h" -#include "cachelib/allocator/tests/TestBase.h" #include "cachelib/allocator/DynamicFreeThresholdStrategy.h" #include "cachelib/allocator/FreeThresholdStrategy.h" +#include "cachelib/allocator/MemoryTierCacheConfig.h" #include "cachelib/allocator/PromotionStrategy.h" - -#include +#include "cachelib/allocator/tests/TestBase.h" namespace facebook { namespace cachelib { @@ -32,18 +32,17 @@ namespace tests { template class AllocatorMemoryTiersTest : public AllocatorTest { private: - template + template void testMultiTiersAsyncOpDuringMove(std::unique_ptr& alloc, - PoolId& pool, bool& quit, MvCallback&& moveCb) { + PoolId& pool, + bool& quit, + MvCallback&& moveCb) { typename AllocatorT::Config config; config.setCacheSize(4 * Slab::kSize); config.enableCachePersistence("/tmp"); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}), - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0}), + MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0})}); config.enableMovingOnSlabRelease(moveCb, {} /* ChainedItemsMoveSync */, -1 /* movingAttemptsLimit */); @@ -53,22 +52,23 @@ class AllocatorMemoryTiersTest : public AllocatorTest { pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); int i = 0; - while(!quit) { - auto handle = alloc->allocate(pool, std::to_string(++i), std::string("value").size()); + while (!quit) { + auto handle = alloc->allocate(pool, std::to_string(++i), + std::string("value").size()); ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } } + public: void testMultiTiersFormFileInvalid() { typename AllocatorT::Config config; config.setCacheSize(100 * Slab::kSize); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromFile("/tmp/a" + std::to_string(::getpid())) - .setRatio(1), - MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) - .setRatio(1) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromFile("/tmp/a" + std::to_string(::getpid())) + .setRatio(1), + MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) + .setRatio(1)}); // More than one tier is not supported ASSERT_THROW(std::make_unique(AllocatorT::SharedMemNew, config), @@ -80,42 +80,47 @@ class AllocatorMemoryTiersTest : public AllocatorTest { config.setCacheSize(100 * Slab::kSize); config.enableCachePersistence("/tmp"); config.usePosixForShm(); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromFile("/tmp/a" + std::to_string(::getpid())) - .setRatio(1), - MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) - .setRatio(1) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromFile("/tmp/a" + std::to_string(::getpid())) + .setRatio(1), + MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) + .setRatio(1)}); auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); ASSERT(alloc != nullptr); - auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); + auto pool = + alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); auto handle = alloc->allocate(pool, "key", std::string("value").size()); ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } - - void testMultiTiersBackgroundMovers() { + + void testMultiTiersBackgroundMovers(bool isDynamicStrategy = false) { typename AllocatorT::Config config; config.setCacheSize(10 * Slab::kSize); config.enableCachePersistence("/tmp"); config.usePosixForShm(); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromShm() - .setRatio(1), - MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) - .setRatio(1) - }); - - config.enableBackgroundEvictor(std::make_shared(2, 10, 100, 40), - std::chrono::milliseconds(10),1); - config.enableBackgroundEvictor(std::make_shared(2, 10, 100, 40), - std::chrono::milliseconds(10),1); - config.enableBackgroundPromoter(std::make_shared(5, 4, 2), - std::chrono::milliseconds(10),1); - - auto allocator = std::make_unique(AllocatorT::SharedMemNew, config); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1), + MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) + .setRatio(1)}); + + if (isDynamicStrategy) { + config.enableBackgroundEvictor( + std::make_shared(2, 10, 100, 40), + std::chrono::milliseconds(10), 1); + } else { + config.enableBackgroundEvictor( + std::make_shared(2, 10, 100, 40), + std::chrono::milliseconds(10), 1); + } + config.enableBackgroundPromoter( + std::make_shared(5, 4, 2), + std::chrono::milliseconds(10), 1); + + auto allocator = + std::make_unique(AllocatorT::SharedMemNew, config); ASSERT(allocator != nullptr); const size_t numBytes = allocator->getCacheMemoryStats().cacheSize; @@ -124,34 +129,36 @@ class AllocatorMemoryTiersTest : public AllocatorTest { const unsigned int keyLen = 100; std::vector sizes = {100}; this->fillUpPoolUntilEvictions(*allocator, poolId, sizes, keyLen); - + const auto key = this->getRandomNewKey(*allocator, keyLen); auto handle = util::allocateAccessible(*allocator, poolId, key, sizes[0]); ASSERT_NE(nullptr, handle); - + const uint8_t cid = allocator->getAllocInfo(handle->getMemory()).classId; auto stats = allocator->getGlobalCacheStats(); - auto slabStats = allocator->getAllocationClassStats(0,0,cid); - const auto& mpStats = allocator->getPoolByTid(poolId, 0).getStats(); - //cache is 10MB should move about 1MB to reach 10% free - uint32_t approxEvict = (1024*1024)/mpStats.acStats.at(cid).allocSize; - while (stats.evictionStats.numMovedItems < approxEvict*0.95 && slabStats.approxFreePercent >= 9.5) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - stats = allocator->getGlobalCacheStats(); - slabStats = allocator->getAllocationClassStats(0,0,cid); + auto slabStats = allocator->getAllocationClassStats(0, 0, cid); + const auto& mpStats = allocator->getPoolByTid(poolId, 0).getStats(); + // cache is 10MB should move about 1MB to reach 10% free + uint32_t approxEvict = (1024 * 1024) / mpStats.acStats.at(cid).allocSize; + while (stats.evictionStats.numMovedItems < approxEvict * 0.95 && + slabStats.approxFreePercent >= 9.5) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + stats = allocator->getGlobalCacheStats(); + slabStats = allocator->getAllocationClassStats(0, 0, cid); } - ASSERT_GE(slabStats.approxFreePercent,9.5); + ASSERT_GE(slabStats.approxFreePercent, 9.5); + + auto perclassEstats = + allocator->getBackgroundMoverClassStats(MoverDir::Evict); + auto perclassPstats = + allocator->getBackgroundMoverClassStats(MoverDir::Promote); - auto perclassEstats = allocator->getBackgroundMoverClassStats(MoverDir::Evict); - auto perclassPstats = allocator->getBackgroundMoverClassStats(MoverDir::Promote); + ASSERT_GE(stats.evictionStats.numMovedItems, 1); + ASSERT_GE(stats.evictionStats.runCount, 1); + ASSERT_GE(stats.promotionStats.numMovedItems, 1); - ASSERT_GE(stats.evictionStats.numMovedItems,1); - ASSERT_GE(stats.evictionStats.runCount,1); - ASSERT_GE(stats.promotionStats.numMovedItems,1); - ASSERT_GE(perclassEstats[0][0][cid], 1); ASSERT_GE(perclassPstats[1][0][cid], 1); - } void testMultiTiersValidMixed() { @@ -159,17 +166,16 @@ class AllocatorMemoryTiersTest : public AllocatorTest { config.setCacheSize(100 * Slab::kSize); config.enableCachePersistence("/tmp"); config.usePosixForShm(); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromShm() - .setRatio(1), - MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) - .setRatio(1) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1), + MemoryTierCacheConfig::fromFile("/tmp/b" + std::to_string(::getpid())) + .setRatio(1)}); auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); ASSERT(alloc != nullptr); - auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); + auto pool = + alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); auto handle = alloc->allocate(pool, "key", std::string("value").size()); ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); @@ -179,17 +185,15 @@ class AllocatorMemoryTiersTest : public AllocatorTest { typename AllocatorT::Config config; config.setCacheSize(100 * Slab::kSize); config.enableCachePersistence("/tmp"); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}), - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0}), + MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0})}); auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); ASSERT(alloc != nullptr); - auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); + auto pool = + alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); auto handle = alloc->allocate(pool, "key", std::string("value").size()); ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); @@ -200,17 +204,15 @@ class AllocatorMemoryTiersTest : public AllocatorTest { config.setCacheSize(100 * Slab::kSize); config.enableCachePersistence("/tmp"); config.usePosixForShm(); - config.configureMemoryTiers({ - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}), - MemoryTierCacheConfig::fromShm() - .setRatio(1).setMemBind({0}) - }); + config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0}), + MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind({0})}); auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); ASSERT(alloc != nullptr); - auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); + auto pool = + alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize); auto handle = alloc->allocate(pool, "key", std::string("value").size()); ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); @@ -223,18 +225,17 @@ class AllocatorMemoryTiersTest : public AllocatorTest { folly::Latch latch(1); bool quit = false; - auto moveCb = [&] (typename AllocatorT::Item& oldItem, - typename AllocatorT::Item& newItem, - typename AllocatorT::Item* /* parentPtr */) { - + auto moveCb = [&](typename AllocatorT::Item& oldItem, + typename AllocatorT::Item& newItem, + typename AllocatorT::Item* /* parentPtr */) { auto key = oldItem.getKey(); - t = std::make_unique([&](){ - // remove() function is blocked by wait context - // till item is moved to next tier. So that, we should - // notify latch before calling remove() - latch.count_down(); - alloc->remove(key); - }); + t = std::make_unique([&]() { + // remove() function is blocked by wait context + // till item is moved to next tier. So that, we should + // notify latch before calling remove() + latch.count_down(); + alloc->remove(key); + }); // wait till async thread is running latch.wait(); memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize()); @@ -253,22 +254,23 @@ class AllocatorMemoryTiersTest : public AllocatorTest { folly::Latch latch(1); bool quit = false; - auto moveCb = [&] (typename AllocatorT::Item& oldItem, - typename AllocatorT::Item& newItem, - typename AllocatorT::Item* /* parentPtr */) { + auto moveCb = [&](typename AllocatorT::Item& oldItem, + typename AllocatorT::Item& newItem, + typename AllocatorT::Item* /* parentPtr */) { auto key = oldItem.getKey(); - if(!quit) { + if (!quit) { // we need to replace only once because subsequent allocate calls // will cause evictions recursevly quit = true; - t = std::make_unique([&](){ - auto handle = alloc->allocate(pool, key, std::string("new value").size()); - // insertOrReplace() function is blocked by wait context - // till item is moved to next tier. So that, we should - // notify latch before calling insertOrReplace() - latch.count_down(); - ASSERT_NO_THROW(alloc->insertOrReplace(handle)); - }); + t = std::make_unique([&]() { + auto handle = + alloc->allocate(pool, key, std::string("new value").size()); + // insertOrReplace() function is blocked by wait context + // till item is moved to next tier. So that, we should + // notify latch before calling insertOrReplace() + latch.count_down(); + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + }); // wait till async thread is running latch.wait(); } diff --git a/cachelib/cachebench/util/CacheConfig.cpp b/cachelib/cachebench/util/CacheConfig.cpp index e1f7040229..87582edd3a 100644 --- a/cachelib/cachebench/util/CacheConfig.cpp +++ b/cachelib/cachebench/util/CacheConfig.cpp @@ -16,12 +16,12 @@ #include "cachelib/cachebench/util/CacheConfig.h" -#include "cachelib/allocator/HitsPerSlabStrategy.h" -#include "cachelib/allocator/LruTailAgeStrategy.h" -#include "cachelib/allocator/RandomStrategy.h" #include "cachelib/allocator/DynamicFreeThresholdStrategy.h" #include "cachelib/allocator/FreeThresholdStrategy.h" +#include "cachelib/allocator/HitsPerSlabStrategy.h" +#include "cachelib/allocator/LruTailAgeStrategy.h" #include "cachelib/allocator/PromotionStrategy.h" +#include "cachelib/allocator/RandomStrategy.h" namespace facebook { namespace cachelib { @@ -99,7 +99,6 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, customConfigJson); - JSONSetVal(configJson, lowEvictionAcWatermark); JSONSetVal(configJson, highEvictionAcWatermark); JSONSetVal(configJson, highEvictionDelta); @@ -121,7 +120,8 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { if (configJson.count("memoryTiers")) { for (auto& it : configJson["memoryTiers"]) { - memoryTierConfigs.push_back(MemoryTierConfig(it).getMemoryTierCacheConfig()); + memoryTierConfigs.push_back( + MemoryTierConfig(it).getMemoryTierCacheConfig()); } } // if you added new fields to the configuration, update the JSONSetVal @@ -137,7 +137,6 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { } } - std::shared_ptr CacheConfig::getRebalanceStrategy() const { if (poolRebalanceIntervalSec == 0) { return nullptr; @@ -158,7 +157,6 @@ std::shared_ptr CacheConfig::getRebalanceStrategy() const { } } - MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, file); JSONSetVal(configJson, ratio); @@ -166,68 +164,68 @@ MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) { checkCorrectSize(); } -std::shared_ptr CacheConfig::getBackgroundEvictorStrategy() const { +std::shared_ptr +CacheConfig::getBackgroundEvictorStrategy() const { if (backgroundEvictorIntervalMilSec == 0) { return nullptr; } - //return std::make_shared(lowEvictionAcWatermark, highEvictionAcWatermark, maxEvictionBatch, minEvictionBatch,highEvictionDelta); - if (backgroundEvictorStrategy=="dynamic") - { - return std::make_shared( - lowEvictionAcWatermark, highEvictionAcWatermark, maxEvictionBatch, - minEvictionBatch,highEvictionDelta); - } - else{ + if (backgroundEvictorStrategy == "dynamic") { + return std::make_shared( + lowEvictionAcWatermark, highEvictionAcWatermark, maxEvictionBatch, + minEvictionBatch, highEvictionDelta); + } + // Default strategy return std::make_shared( - lowEvictionAcWatermark, highEvictionAcWatermark, - maxEvictionBatch, minEvictionBatch); - } + lowEvictionAcWatermark, highEvictionAcWatermark, maxEvictionBatch, + minEvictionBatch); } -std::shared_ptr CacheConfig::getBackgroundPromoterStrategy() const { +std::shared_ptr +CacheConfig::getBackgroundPromoterStrategy() const { if (backgroundPromoterIntervalMilSec == 0) { return nullptr; } - return std::make_shared(promotionAcWatermark, maxPromotionBatch, minPromotionBatch); + return std::make_shared( + promotionAcWatermark, maxPromotionBatch, minPromotionBatch); } -static bool starts_with() {return true;} +static bool starts_with() { return true; } std::vector MemoryTierConfig::parseNumaNodes() { std::vector numaNodes; std::vector tokens; folly::split(",", memBindNodes, tokens, true /*ignore empty*/); - for(const auto &token : tokens) { - if(token.startsWith("!")) { - throw std::invalid_argument(folly::sformat( - "invalid NUMA nodes binding in memory tier config: {} " - "inverse !N or !N-N is not supported " - "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.", - token)); - } - else if(token.startsWith("+")) { - throw std::invalid_argument(folly::sformat( - "invalid NUMA nodes binding in memory tier config: {} " - "relative nodes are not supported. " - "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.", - token)); - } - else if (token.contains("-")) { + for (const auto& token : tokens) { + if (token.startsWith("!")) { + throw std::invalid_argument( + folly::sformat("invalid NUMA nodes binding in memory tier config: {} " + "inverse !N or !N-N is not supported " + "nodes may be specified as N,N,N or N-N or N,N-N or " + "N-N,N-N and so forth.", + token)); + } else if (token.startsWith("+")) { + throw std::invalid_argument( + folly::sformat("invalid NUMA nodes binding in memory tier config: {} " + "relative nodes are not supported. " + "nodes may be specified as N,N,N or N-N or N,N-N or " + "N-N,N-N and so forth.", + token)); + } else if (token.contains("-")) { size_t begin, end; - if(folly::split("-", token, begin, end) && begin < end) { - while(begin <=end) { + if (folly::split("-", token, begin, end) && begin < end) { + while (begin <= end) { numaNodes.push_back(begin++); } } else { throw std::invalid_argument(folly::sformat( - "invalid NUMA nodes binding in memory tier config: {} " - "Invalid range format. " - "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.", - token)); + "invalid NUMA nodes binding in memory tier config: {} " + "Invalid range format. " + "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so " + "forth.", + token)); } - } - else { + } else { numaNodes.push_back(folly::to(token)); } } diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 8b076f8fd7..e8ae4239c2 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -18,9 +18,9 @@ #include +#include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/allocator/CacheAllocator.h" #include "cachelib/allocator/RebalanceStrategy.h" -#include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/cachebench/util/JSONConfig.h" #include "cachelib/common/Ticker.h" #include "cachelib/navy/common/Device.h" @@ -57,7 +57,7 @@ struct MemoryTierConfig : public JSONConfig { size_t ratio{0}; std::string memBindNodes{""}; -private: + private: MemoryTierCacheConfig memoryTierCacheConfigFromSource() { if (file.empty()) { return MemoryTierCacheConfig::fromShm(); @@ -269,17 +269,18 @@ struct CacheConfig : public JSONConfig { double maxAcAllocationWatermark{0.0}; double numDuplicateElements{0.0}; // inclusivness of the cache - double syncPromotion{0.0}; // can promotion be done synchronously in user thread - + double syncPromotion{0.0}; // can promotion be done synchronously in user + // thread + uint64_t evictorThreads{1}; uint64_t promoterThreads{1}; - + uint64_t maxEvictionBatch{40}; uint64_t maxPromotionBatch{10}; - + uint64_t minEvictionBatch{5}; uint64_t minPromotionBatch{5}; - + uint64_t maxEvictionPromotionHotness{60}; // @@ -314,7 +315,8 @@ struct CacheConfig : public JSONConfig { std::shared_ptr getRebalanceStrategy() const; std::shared_ptr getBackgroundEvictorStrategy() const; - std::shared_ptr getBackgroundPromoterStrategy() const; + std::shared_ptr getBackgroundPromoterStrategy() + const; }; } // namespace cachebench } // namespace cachelib