diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index c8c11c77f5..a9f27727da 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -27,11 +27,12 @@ CacheAllocator::CacheAllocator(Config config) memoryTierConfigs(config.getMemoryTierConfigs()), isOnShm_{config.memMonitoringEnabled()}, config_(config.validate()), - tempShm_(isOnShm_ ? std::make_unique( - config_.getCacheSize()) - : nullptr), + tempShm_(isOnShm_ + ? std::make_unique(config_.getCacheSize()) + : nullptr), allocator_(createPrivateAllocator()), - compactCacheManager_(std::make_unique(*allocator_[0] /* TODO */)), + compactCacheManager_( + std::make_unique(*allocator_[0] /* TODO */)), compressor_(createPtrCompressor()), mmContainers_(numTiers_), accessContainer_(std::make_unique( @@ -47,12 +48,11 @@ CacheAllocator::CacheAllocator(Config config) movesMap_(kShards), moveLock_(kShards), cacheCreationTime_{util::getCurrentTimeSec()} { - if (numTiers_ > 1 || std::holds_alternative( - memoryTierConfigs[0].getShmTypeOpts())) { + memoryTierConfigs[0].getShmTypeOpts())) { throw std::runtime_error( - "Using custom memory tier or using more than one tier is only " - "supported for Shared Memory."); + "Using custom memory tier or using more than one tier is only " + "supported for Shared Memory."); } initCommon(false); } @@ -64,12 +64,10 @@ CacheAllocator::createPrivateAllocator() { if (isOnShm_) allocators.emplace_back(std::make_unique( - getAllocatorConfig(config_), - tempShm_->getAddr(), - config_.size)); + getAllocatorConfig(config_), tempShm_->getAddr(), config_.size)); else allocators.emplace_back(std::make_unique( - getAllocatorConfig(config_), config_.size)); + getAllocatorConfig(config_), config_.size)); return allocators; } @@ -100,10 +98,11 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) memoryTierConfigs(config.getMemoryTierConfigs()), isOnShm_{true}, config_(config.validate()), - shmManager_( - std::make_unique(config_.cacheDir, config_.isUsingPosixShm())), + shmManager_(std::make_unique(config_.cacheDir, + config_.isUsingPosixShm())), allocator_(createAllocators()), - compactCacheManager_(std::make_unique(*allocator_[0] /* TODO */)), + compactCacheManager_( + std::make_unique(*allocator_[0] /* TODO */)), compressor_(createPtrCompressor()), mmContainers_(numTiers_), accessContainer_(std::make_unique( @@ -114,7 +113,8 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) config_.accessConfig.getNumBuckets()), nullptr, ShmSegmentOpts(config_.accessConfig.getPageSize(), - false, config_.isUsingPosixShm())) + false, + config_.isUsingPosixShm())) .addr, compressor_, [this](Item* it) -> ItemHandle { return acquire(it); })), @@ -126,7 +126,8 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) config_.chainedItemAccessConfig.getNumBuckets()), nullptr, ShmSegmentOpts(config_.accessConfig.getPageSize(), - false, config_.isUsingPosixShm())) + false, + config_.isUsingPosixShm())) .addr, compressor_, [this](Item* it) -> ItemHandle { return acquire(it); })), @@ -137,7 +138,7 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) cacheCreationTime_{util::getCurrentTimeSec()} { initCommon(false); shmManager_->removeShm(detail::kShmInfoName, - PosixSysVSegmentOpts(config_.isUsingPosixShm())); + PosixSysVSegmentOpts(config_.isUsingPosixShm())); } template @@ -157,15 +158,21 @@ CacheAllocator::CacheAllocator(SharedMemAttachT, Config config) accessContainer_(std::make_unique( deserializer_->deserialize(), config_.accessConfig, - shmManager_->attachShm(detail::kShmHashTableName, nullptr, - ShmSegmentOpts(PageSizeT::NORMAL, false, config_.isUsingPosixShm())), + shmManager_->attachShm(detail::kShmHashTableName, + nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, + false, + config_.isUsingPosixShm())), compressor_, [this](Item* it) -> ItemHandle { return acquire(it); })), chainedItemAccessContainer_(std::make_unique( deserializer_->deserialize(), config_.chainedItemAccessConfig, - shmManager_->attachShm(detail::kShmChainedItemHashTableName, nullptr, - ShmSegmentOpts(PageSizeT::NORMAL, false, config_.isUsingPosixShm())), + shmManager_->attachShm(detail::kShmChainedItemHashTableName, + nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, + false, + config_.isUsingPosixShm())), compressor_, [this](Item* it) -> ItemHandle { return acquire(it); })), chainedItemLocks_(config_.chainedItemsLockPower, @@ -184,7 +191,7 @@ CacheAllocator::CacheAllocator(SharedMemAttachT, Config config) // this info shm segment here and the new info shm segment's size is larger // than this one, creating new one will fail. shmManager_->removeShm(detail::kShmInfoName, - PosixSysVSegmentOpts(config_.isUsingPosixShm())); + PosixSysVSegmentOpts(config_.isUsingPosixShm())); } template @@ -226,14 +233,15 @@ CacheAllocator::restoreMemoryAllocator(TierId tid) { deserializer_->deserialize(), shmManager_ ->attachShm(detail::kShmCacheName + std::to_string(tid), - config_.slabMemoryBaseAddr, createShmCacheOpts(tid)).addr, + config_.slabMemoryBaseAddr, createShmCacheOpts(tid)) + .addr, memoryTierConfigs[tid].getSize(), config_.disableFullCoredump); } template -std::unique_ptr -CacheAllocator::restoreCCacheManager(TierId tid) { +std::unique_ptr CacheAllocator::restoreCCacheManager( + TierId tid) { return std::make_unique( deserializer_->deserialize(), *allocator_[tid]); @@ -268,8 +276,9 @@ void CacheAllocator::initNvmCache(bool dramCacheAttached) { return; } - nvmCacheState_.emplace(NvmCacheState(config_.cacheDir, config_.isNvmCacheEncryptionEnabled(), - config_.isNvmCacheTruncateAllocSizeEnabled())); + nvmCacheState_.emplace( + NvmCacheState(config_.cacheDir, config_.isNvmCacheEncryptionEnabled(), + config_.isNvmCacheTruncateAllocSizeEnabled())); // for some usecases that create pools, restoring nvmcache when dram cache // is not persisted is not supported. @@ -329,8 +338,9 @@ void CacheAllocator::initWorkers() { template std::unique_ptr CacheAllocator::createDeserializer() { - auto infoAddr = shmManager_->attachShm(detail::kShmInfoName, nullptr, - ShmSegmentOpts(PageSizeT::NORMAL, false, config_.isUsingPosixShm())); + auto infoAddr = shmManager_->attachShm( + detail::kShmInfoName, nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, false, config_.isUsingPosixShm())); return std::make_unique( reinterpret_cast(infoAddr.addr), reinterpret_cast(infoAddr.addr) + infoAddr.size); @@ -353,11 +363,11 @@ CacheAllocator::allocate(PoolId poolId, template typename CacheAllocator::ItemHandle CacheAllocator::allocateInternalTier(TierId tid, - PoolId pid, - typename Item::Key key, - uint32_t size, - uint32_t creationTime, - uint32_t expiryTime) { + PoolId pid, + typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -373,7 +383,8 @@ CacheAllocator::allocateInternalTier(TierId tid, void* memory = allocator_[tid]->allocate(pid, requiredSize); // TODO: Today disableEviction means do not evict from memory (DRAM). - // Should we support eviction between memory tiers (e.g. from DRAM to PMEM)? + // Should we support eviction between memory tiers (e.g. from DRAM to + // PMEM)? if (memory == nullptr && !config_.disableEviction) { memory = findEviction(tid, pid, cid); } @@ -396,7 +407,7 @@ CacheAllocator::allocateInternalTier(TierId tid, util::getFragmentation(*this, *handle)); } - } else { // failed to allocate memory. + } else { // failed to allocate memory. (*stats_.allocFailures)[pid][cid].inc(); // TODO: per-tier // wake up rebalancer if (poolRebalancer_) { @@ -422,9 +433,11 @@ CacheAllocator::allocateInternal(PoolId pid, uint32_t creationTime, uint32_t expiryTime) { auto tid = 0; /* TODO: consult admission policy */ - for(TierId tid = 0; tid < numTiers_; ++tid) { - auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); - if (handle) return handle; + for (TierId tid = 0; tid < numTiers_; ++tid) { + auto handle = + allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); + if (handle) + return handle; } return {}; } @@ -998,8 +1011,8 @@ bool CacheAllocator::replaceInMMContainer(Item* oldItem, } template -bool CacheAllocator::replaceInMMContainer(EvictionIterator& oldItemIt, - Item& newItem) { +bool CacheAllocator::replaceInMMContainer( + EvictionIterator& oldItemIt, Item& newItem) { auto& oldContainer = getMMContainer(*oldItemIt); auto& newContainer = getMMContainer(newItem); @@ -1188,8 +1201,8 @@ bool CacheAllocator::addWaitContextForMovingItem( template template typename CacheAllocator::ItemHandle -CacheAllocator::moveRegularItemOnEviction( - ItemPtr& oldItemPtr, ItemHandle& newItemHdl) { +CacheAllocator::moveRegularItemOnEviction(ItemPtr& oldItemPtr, + ItemHandle& newItemHdl) { // TODO: should we introduce new latency tracker. E.g. evictRegularLatency_ // ??? util::LatencyTracker tracker{stats_.evictRegularLatency_}; @@ -1463,16 +1476,17 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { // for chained items, the ownership of the parent can change. We try to // evict what we think as parent and see if the eviction of parent // recycles the child we intend to. - + ItemHandle toReleaseHandle = tryEvictToNextMemoryTier(tid, pid, itr); bool movedToNextTier = false; - if(toReleaseHandle) { + if (toReleaseHandle) { movedToNextTier = true; } else { toReleaseHandle = itr->isChainedItem() ? advanceIteratorAndTryEvictChainedItem(tid, pid, itr) - : advanceIteratorAndTryEvictRegularItem(tid, pid, mmContainer, itr); + : advanceIteratorAndTryEvictRegularItem(tid, pid, mmContainer, + itr); } if (toReleaseHandle) { @@ -1569,19 +1583,20 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( template template typename CacheAllocator::ItemHandle -CacheAllocator::tryEvictToNextMemoryTier( - TierId tid, PoolId pid, ItemPtr& item) { - if(item->isChainedItem()) return {}; // TODO: We do not support ChainedItem yet - if(item->isExpired()) return acquire(item); +CacheAllocator::tryEvictToNextMemoryTier(TierId tid, + PoolId pid, + ItemPtr& item) { + if (item->isChainedItem()) + return {}; // TODO: We do not support ChainedItem yet + if (item->isExpired()) + return acquire(item); TierId nextTier = tid; // TODO - calculate this based on some admission policy while (++nextTier < numTiers_) { // try to evict down to the next memory tiers // allocateInternal might trigger another eviction - auto newItemHdl = allocateInternalTier(nextTier, pid, - item->getKey(), - item->getSize(), - item->getCreationTime(), - item->getExpiryTime()); + auto newItemHdl = + allocateInternalTier(nextTier, pid, item->getKey(), item->getSize(), + item->getCreationTime(), item->getExpiryTime()); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item->getSize()); @@ -1935,14 +1950,12 @@ void CacheAllocator::invalidateNvm(Item& item) { } template -TierId -CacheAllocator::getTierId(const Item& item) const { +TierId CacheAllocator::getTierId(const Item& item) const { return getTierId(item.getMemory()); } template -TierId -CacheAllocator::getTierId(const void* ptr) const { +TierId CacheAllocator::getTierId(const void* ptr) const { for (TierId tid = 0; tid < numTiers_; tid++) { if (allocator_[tid]->isMemoryInAllocator(ptr)) return tid; @@ -2160,7 +2173,7 @@ CacheAllocator::getSampleItem() { template std::vector CacheAllocator::dumpEvictionIterator( - PoolId pid, ClassId cid, size_t numItems) { + PoolId pid, ClassId cid, size_t numItems) { if (numItems == 0) { return {}; } @@ -2171,8 +2184,8 @@ std::vector CacheAllocator::dumpEvictionIterator( if (static_cast(tid) >= mmContainers_.size() || static_cast(pid) >= mmContainers_[tid].size() || static_cast(cid) >= mmContainers_[tid][pid].size()) { - throw std::invalid_argument( - folly::sformat("Invalid TierId: {} and PoolId: {} and ClassId: {}.", tid, pid, cid)); + throw std::invalid_argument(folly::sformat( + "Invalid TierId: {} and PoolId: {} and ClassId: {}.", tid, pid, cid)); } std::vector content; @@ -2365,12 +2378,50 @@ PoolId CacheAllocator::addPool( folly::SharedMutex::WriteHolder w(poolsResizeAndRebalanceLock_); PoolId pid = 0; + size_t remainingPoolSize = size; + std::vector tierPoolSizes; + std::vector tierPoolRoundOffs(numTiers_, 0); auto tierConfigs = config_.getMemoryTierConfigs(); + size_t totalCacheSize = 0; + + for (TierId tid = 0; tid < numTiers_; tid++) { + totalCacheSize += allocator_[tid]->getMemorySize(); + } + + size_t remainingCacheSize = totalCacheSize; + for (TierId tid = 0; tid < numTiers_; tid++) { + auto tierSizeRatio = + static_cast(allocator_[tid]->getMemorySize()) / totalCacheSize; + size_t tierPoolSize = static_cast(tierSizeRatio * size); + tierPoolSizes.push_back(tierPoolSize); + remainingPoolSize -= tierPoolSizes.back(); + remainingCacheSize -= tierPoolSizes.back(); + } + + if (remainingPoolSize) { + if (remainingPoolSize < remainingCacheSize) { + for (TierId tid = 0; (tid < numTiers_) && remainingPoolSize; + tid++, remainingPoolSize--) { + tierPoolRoundOffs[tid]++; + } + } else { + throw std::invalid_argument(folly::sformat( + "Not enough memory to allocate pool of size {}", size)); + } + } + for (TierId tid = 0; tid < numTiers_; tid++) { - auto tierSizeRatio = static_cast( - tierConfigs[tid].getSize()) / config_.getCacheSize(); - auto tierPoolSize = static_cast(tierSizeRatio * size); - auto res = allocator_[tid]->addPool(name, tierPoolSize, allocSizes, ensureProvisionable); + auto res = + allocator_[tid]->addPool(name, tierPoolSizes[tid], allocSizes, + ensureProvisionable, &tierPoolRoundOffs[tid]); + if (tierPoolRoundOffs[tid]) { + if (tid + 1 < numTiers_) { + tierPoolRoundOffs[tid + 1] += tierPoolRoundOffs[tid]; + } else { + throw std::invalid_argument(folly::sformat( + "Not enough memory to allocate pool of size {}", size)); + } + } XDCHECK(tid == 0 || res == pid); pid = res; } @@ -2409,12 +2460,14 @@ void CacheAllocator::overridePoolOptimizeStrategy( } template -void CacheAllocator::overridePoolConfig(TierId tid, PoolId pid, +void CacheAllocator::overridePoolConfig(TierId tid, + PoolId pid, const MMConfig& config) { // TODO: add generic tier id checking if (static_cast(pid) >= mmContainers_[tid].size()) { - throw std::invalid_argument(folly::sformat( - "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_[tid].size())); + throw std::invalid_argument( + folly::sformat("Invalid PoolId: {}, size of pools: {}", pid, + mmContainers_[tid].size())); } auto& pool = allocator_[tid]->getPool(pid); for (unsigned int cid = 0; cid < pool.getNumClassId(); ++cid) { @@ -2524,7 +2577,8 @@ std::set CacheAllocator::getRegularPoolIdsForResize() // only after all slabs are allocated. return (allocator_[currentTier()]->allSlabsAllocated()) || (allocator_[currentTier()]->getAdvisedMemorySize() != 0) - ? filterCompactCachePools(allocator_[currentTier()]->getPoolsOverLimit()) + ? filterCompactCachePools( + allocator_[currentTier()]->getPoolsOverLimit()) : std::set{}; } @@ -2533,6 +2587,16 @@ const std::string CacheAllocator::getCacheName() const { return config_.cacheName; } +template +size_t CacheAllocator::getPoolSize(PoolId poolId) const { + size_t poolSize = 0; + for (auto& allocator : allocator_) { + const auto& pool = allocator->getPool(poolId); + poolSize += pool.getPoolSize(); + } + return poolSize; +} + template PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { const auto& pool = allocator_[currentTier()]->getPool(poolId); @@ -2589,8 +2653,10 @@ PoolEvictionAgeStats CacheAllocator::getPoolEvictionAgeStats( const auto& allocSizes = pool.getAllocSizes(); for (ClassId cid = 0; cid < static_cast(allocSizes.size()); ++cid) { auto& mmContainer = getMMContainer(currentTier(), pid, cid); - const auto numItemsPerSlab = - allocator_[currentTier()]->getPool(pid).getAllocationClass(cid).getAllocsPerSlab(); + const auto numItemsPerSlab = allocator_[currentTier()] + ->getPool(pid) + .getAllocationClass(cid) + .getAllocsPerSlab(); const auto projectionLength = numItemsPerSlab * slabProjectionLength; stats.classEvictionAgeStats[cid] = mmContainer.getEvictionAgeStat(projectionLength); @@ -2661,7 +2727,8 @@ void CacheAllocator::releaseSlab(PoolId pid, } template -SlabReleaseStats CacheAllocator::getSlabReleaseStats() const noexcept { +SlabReleaseStats CacheAllocator::getSlabReleaseStats() + const noexcept { std::lock_guard l(workersMutex_); return SlabReleaseStats{stats_.numActiveSlabReleases.get(), stats_.numReleasedForRebalance.get(), @@ -2678,8 +2745,8 @@ SlabReleaseStats CacheAllocator::getSlabReleaseStats() const noexcep } template -void CacheAllocator::releaseSlabImpl(TierId tid, - const SlabReleaseContext& releaseContext) { +void CacheAllocator::releaseSlabImpl( + TierId tid, const SlabReleaseContext& releaseContext) { util::Throttler throttler(config_.throttleConfig); // Active allocations need to be freed before we can release this slab @@ -2851,17 +2918,17 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { return newItemHdl; } - const auto allocInfo = - allocator_[getTierId(oldItem)]->getAllocInfo(static_cast(&oldItem)); + const auto allocInfo = allocator_[getTierId(oldItem)]->getAllocInfo( + static_cast(&oldItem)); // Set up the destination for the move. Since oldItem would have the moving // bit set, it won't be picked for eviction. auto newItemHdl = allocateInternalTier(getTierId(oldItem), - allocInfo.poolId, - oldItem.getKey(), - oldItem.getSize(), - oldItem.getCreationTime(), - oldItem.getExpiryTime()); + allocInfo.poolId, + oldItem.getKey(), + oldItem.getSize(), + oldItem.getCreationTime(), + oldItem.getExpiryTime()); if (!newItemHdl) { return {}; } @@ -2941,8 +3008,8 @@ void CacheAllocator::evictForSlabRelease( // we managed to evict the corresponding owner of the item and have the // last handle for the owner. if (owningHandle) { - const auto allocInfo = - allocator_[getTierId(item)]->getAllocInfo(static_cast(&item)); + const auto allocInfo = allocator_[getTierId(item)]->getAllocInfo( + static_cast(&item)); if (owningHandle->hasChainedItem()) { (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId] .inc(); @@ -3001,7 +3068,8 @@ CacheAllocator::evictNormalItemForSlabRelease(Item& item) { } auto evictHandle = tryEvictToNextMemoryTier(&item); - if(evictHandle) return evictHandle; + if (evictHandle) + return evictHandle; auto predicate = [](const Item& it) { return it.getRefCount() == 0; }; @@ -3154,7 +3222,6 @@ bool CacheAllocator::removeIfExpired(const ItemHandle& handle) { template bool CacheAllocator::markMovingForSlabRelease( const SlabReleaseContext& ctx, void* alloc, util::Throttler& throttler) { - // MemoryAllocator::processAllocForRelease will execute the callback // if the item is not already free. So there are three outcomes here: // 1. Item not freed yet and marked as moving @@ -3169,7 +3236,8 @@ bool CacheAllocator::markMovingForSlabRelease( bool itemFreed = true; bool markedMoving = false; TierId tid = 0; - const auto fn = [&markedMoving, &itemFreed, &tid, this /* TODO - necessary for getTierId */](void* memory) { + const auto fn = [&markedMoving, &itemFreed, &tid, + this /* TODO - necessary for getTierId */](void* memory) { // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); @@ -3221,7 +3289,8 @@ CCacheT* CacheAllocator::addCompactCache(folly::StringPiece name, size_t size, Args&&... args) { if (numTiers_ != 1) - throw std::runtime_error("TODO: compact cache for multi-tier Cache not supported."); + throw std::runtime_error( + "TODO: compact cache for multi-tier Cache not supported."); if (!config_.isCompactCacheEnabled()) { throw std::logic_error("Compact cache is not enabled"); @@ -3340,7 +3409,7 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { // TODO: implement serialization for multiple tiers auto serializeMMContainers = [](MMContainers& mmContainers) { MMSerializationTypeContainer state; - for (unsigned int i = 0; i < 1 /* TODO: */ ; ++i) { + for (unsigned int i = 0; i < 1 /* TODO: */; ++i) { for (unsigned int j = 0; j < mmContainers[i].size(); ++j) { for (unsigned int k = 0; k < mmContainers[i][j].size(); ++k) { if (mmContainers[i][j][k]) { @@ -3356,7 +3425,8 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { AccessSerializationType accessContainerState = accessContainer_->saveState(); // TODO: foreach allocator - MemoryAllocator::SerializationType allocatorState = allocator_[0]->saveState(); + MemoryAllocator::SerializationType allocatorState = + allocator_[0]->saveState(); CCacheManager::SerializationType ccState = compactCacheManager_->saveState(); AccessSerializationType chainedItemAccessContainerState = @@ -3471,8 +3541,10 @@ void CacheAllocator::saveRamCache() { ShmSegmentOpts opts; opts.typeOpts = PosixSysVSegmentOpts(config_.isUsingPosixShm()); - void* infoAddr = shmManager_->createShm(detail::kShmInfoName, ioBuf->length(), - nullptr, opts).addr; + void* infoAddr = + shmManager_ + ->createShm(detail::kShmInfoName, ioBuf->length(), nullptr, opts) + .addr; Serializer serializer(reinterpret_cast(infoAddr), reinterpret_cast(infoAddr) + ioBuf->length()); serializer.writeToBuffer(std::move(ioBuf)); @@ -3669,7 +3741,7 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { template CacheMemoryStats CacheAllocator::getCacheMemoryStats() const { size_t totalCacheSize = 0; - for(auto& allocator: allocator_) { + for (auto& allocator : allocator_) { totalCacheSize += allocator->getMemorySize(); } @@ -3824,8 +3896,8 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { } template -bool CacheAllocator::cleanupStrayShmSegments( - const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { +bool CacheAllocator< + CacheTrait>::cleanupStrayShmSegments(const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) { try { // cache dir exists. clean up only if there are no other processes @@ -3842,8 +3914,8 @@ bool CacheAllocator::cleanupStrayShmSegments( // Any other concurrent process can not be attached to the segments or // even if it does, we want to mark it for destruction. ShmManager::removeByName(cacheDir, detail::kShmInfoName, posix); - ShmManager::removeByName(cacheDir, detail::kShmCacheName - + std::to_string(0), posix); + ShmManager::removeByName(cacheDir, + detail::kShmCacheName + std::to_string(0), posix); ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix); ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName, posix); @@ -3851,7 +3923,8 @@ bool CacheAllocator::cleanupStrayShmSegments( // TODO(SHM_FILE): try to nuke segments of differente types (which require // extra info) // for (auto &tier : config) { - // ShmManager::removeByName(cacheDir, tierShmName, config_.memoryTiers[i].opts); + // ShmManager::removeByName(cacheDir, tierShmName, + // config_.memoryTiers[i].opts); // } } return true; diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 319e66a626..3c58a8654c 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1045,6 +1045,9 @@ class CacheAllocator : public CacheBase { // get cache name const std::string getCacheName() const override final; + // combined pool size for all memory tiers + size_t getPoolSize(PoolId pid) const; + // pool stats by pool id PoolStats getPoolStats(PoolId pid) const override final; diff --git a/cachelib/allocator/memory/MemoryAllocator.cpp b/cachelib/allocator/memory/MemoryAllocator.cpp index 13a0a24b44..d021477fc5 100644 --- a/cachelib/allocator/memory/MemoryAllocator.cpp +++ b/cachelib/allocator/memory/MemoryAllocator.cpp @@ -82,7 +82,8 @@ void* MemoryAllocator::allocateZeroedSlab(PoolId id) { PoolId MemoryAllocator::addPool(folly::StringPiece name, size_t size, const std::set& allocSizes, - bool ensureProvisionable) { + bool ensureProvisionable, + size_t* extraBytes) { const std::set& poolAllocSizes = allocSizes.empty() ? config_.allocSizes : allocSizes; @@ -100,7 +101,8 @@ PoolId MemoryAllocator::addPool(folly::StringPiece name, size)); } - return memoryPoolManager_.createNewPool(name, size, poolAllocSizes); + return memoryPoolManager_.createNewPool(name, size, poolAllocSizes, + extraBytes); } PoolId MemoryAllocator::getPoolId(const std::string& name) const noexcept { diff --git a/cachelib/allocator/memory/MemoryAllocator.h b/cachelib/allocator/memory/MemoryAllocator.h index 4026bf7afb..37d67c89e7 100644 --- a/cachelib/allocator/memory/MemoryAllocator.h +++ b/cachelib/allocator/memory/MemoryAllocator.h @@ -205,7 +205,8 @@ class MemoryAllocator { PoolId addPool(folly::StringPiece name, size_t size, const std::set& allocSizes = {}, - bool ensureProvisionable = false); + bool ensureProvisionable = false, + size_t* extraBytes = nullptr); // shrink the existing pool by _bytes_ . // @param id the id for the pool @@ -515,14 +516,12 @@ class MemoryAllocator { using CompressedPtr = facebook::cachelib::CompressedPtr; template - using PtrCompressor = - facebook::cachelib::PtrCompressor>>; + using PtrCompressor = facebook::cachelib:: + PtrCompressor>>; template using SingleTierPtrCompressor = - facebook::cachelib::PtrCompressor; + facebook::cachelib::PtrCompressor; // compress a given pointer to a valid allocation made out of this allocator // through an allocate() or nullptr. Calling this otherwise with invalid @@ -636,9 +635,9 @@ class MemoryAllocator { // returns ture if ptr points to memory which is managed by this // allocator - bool isMemoryInAllocator(const void *ptr) { - return ptr && ptr >= slabAllocator_.getSlabMemoryBegin() - && ptr < slabAllocator_.getSlabMemoryEnd(); + bool isMemoryInAllocator(const void* ptr) { + return ptr && ptr >= slabAllocator_.getSlabMemoryBegin() && + ptr < slabAllocator_.getSlabMemoryEnd(); } private: diff --git a/cachelib/allocator/memory/MemoryPoolManager.cpp b/cachelib/allocator/memory/MemoryPoolManager.cpp index 30e7551d4d..687c1a01db 100644 --- a/cachelib/allocator/memory/MemoryPoolManager.cpp +++ b/cachelib/allocator/memory/MemoryPoolManager.cpp @@ -90,7 +90,8 @@ size_t MemoryPoolManager::getRemainingSizeLocked() const noexcept { PoolId MemoryPoolManager::createNewPool(folly::StringPiece name, size_t poolSize, - const std::set& allocSizes) { + const std::set& allocSizes, + size_t* extraBytes) { folly::SharedMutex::WriteHolder l(lock_); if (poolsByName_.find(name) != poolsByName_.end()) { throw std::invalid_argument("Duplicate pool"); @@ -109,6 +110,16 @@ PoolId MemoryPoolManager::createNewPool(folly::StringPiece name, poolSize)); } + if (extraBytes && (*extraBytes)) { + if (remaining >= poolSize + *extraBytes) { + poolSize += *extraBytes; + *extraBytes = 0; + } else { + poolSize += (remaining - poolSize); + *extraBytes -= (remaining - poolSize); + } + } + const PoolId id = nextPoolId_; pools_[id].reset(new MemoryPool(id, poolSize, slabAlloc_, allocSizes)); poolsByName_.insert({name.str(), id}); diff --git a/cachelib/allocator/memory/MemoryPoolManager.h b/cachelib/allocator/memory/MemoryPoolManager.h index abae63071e..b58ec1b6b7 100644 --- a/cachelib/allocator/memory/MemoryPoolManager.h +++ b/cachelib/allocator/memory/MemoryPoolManager.h @@ -75,7 +75,8 @@ class MemoryPoolManager { // std::logic_error if we have run out the allowed number of pools. PoolId createNewPool(folly::StringPiece name, size_t size, - const std::set& allocSizes); + const std::set& allocSizes, + size_t* extraBytes = nullptr); // shrink the existing pool by _bytes_ . // @param bytes the number of bytes to be taken away from the pool diff --git a/cachelib/allocator/tests/MemoryTiersTest.cpp b/cachelib/allocator/tests/MemoryTiersTest.cpp index 94339d560b..83c39b922a 100644 --- a/cachelib/allocator/tests/MemoryTiersTest.cpp +++ b/cachelib/allocator/tests/MemoryTiersTest.cpp @@ -26,18 +26,18 @@ namespace tests { using LruAllocatorConfig = CacheAllocatorConfig; using LruMemoryTierConfigs = LruAllocatorConfig::MemoryTierConfigs; using Strings = std::vector; + +constexpr size_t MB = 1024ULL * 1024ULL; +constexpr size_t GB = MB * 1024ULL; + using SizePair = std::tuple; using SizePairs = std::vector; -const size_t defaultTotalCacheSize{1 * 1024 * 1024 * 1024}; +const size_t defaultTotalCacheSize{1 * GB}; const std::string defaultCacheDir{"/var/metadataDir"}; const std::string defaultPmemPath{"/dev/shm/p1"}; const std::string defaultDaxPath{"/dev/dax0.0"}; -const size_t metaDataSize = 4194304; -constexpr size_t MB = 1024ULL * 1024ULL; -constexpr size_t GB = MB * 1024ULL; - template class MemoryTiersTest : public AllocatorTest { public: @@ -124,6 +124,13 @@ class MemoryTiersTest : public AllocatorTest { dramConfig.setCacheSize(totalCacheSize); return dramConfig; } + + void validatePoolSize(PoolId poolId, + std::unique_ptr& allocator, + size_t expectedSize) { + size_t actualSize = allocator->getPoolSize(poolId); + EXPECT_EQ(actualSize, expectedSize); + } }; using LruMemoryTiersTest = MemoryTiersTest; @@ -225,30 +232,57 @@ TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) { std::invalid_argument); } -TEST_F(LruMemoryTiersTest, TestTieredCacheSize) { - size_t totalSizes[] = {50 * MB, 77 * MB, 100 * MB, 101 * MB + MB / 2, - 1 * GB, 4 * GB, 8 * GB, 9 * GB}; - size_t numTiers[] = {2, 3, 4}; - - auto getCacheSize = [&](size_t cacheSize, size_t tiers) { - std::unique_ptr alloc; - if (tiers < 2) { - alloc = std::unique_ptr( - new LruAllocator(createDramCacheConfig(cacheSize))); +TEST_F(LruMemoryTiersTest, TestPoolAllocations) { + std::vector totalCacheSizes = {48 * MB, 51 * MB, 256 * MB, + 1 * GB, 5 * GB, 8 * GB}; + std::vector numTiers = {2, 3, 4}; + std::vector sizePairs = { + {}, + {std::make_tuple(1, 0)}, + {std::make_tuple(3, 0), std::make_tuple(2, 0)}, + {std::make_tuple(2, 0), std::make_tuple(2, 0), std::make_tuple(1, 0)}, + {std::make_tuple(1, 0), std::make_tuple(5, 0), std::make_tuple(1, 0), + std::make_tuple(2, 0)}}; + const std::string path = "/tmp/tier"; + std::vector paths = { + {}, + {path + "0"}, + {path + "0", path + "1"}, + {path + "0", path + "1", path + "2"}, + {path + "0", path + "1", path + "2", path + "3"}}; + + auto testAddPool = [&](LruAllocatorConfig& config, size_t poolSize, + bool isSizeValid = true, bool isTestMaxSize = false) { + if (isTestMaxSize) { + std::unique_ptr alloc = std::unique_ptr( + new LruAllocator(LruAllocator::SharedMemNew, config)); + auto pool = + alloc->addPool("maxPoolSize", alloc->getCacheMemoryStats().cacheSize); + validatePoolSize(pool, alloc, alloc->getCacheMemoryStats().cacheSize); + } + std::unique_ptr alloc = std::unique_ptr( + new LruAllocator(LruAllocator::SharedMemNew, config)); + if (isSizeValid) { + auto pool = alloc->addPool("validPoolSize", poolSize); + validatePoolSize(pool, alloc, poolSize); } else { - alloc = std::unique_ptr( - new LruAllocator(LruAllocator::SharedMemNew, - createTieredCacheConfig(cacheSize, tiers))); + EXPECT_THROW(alloc->addPool("invalidPoolSize", poolSize), + std::invalid_argument); } - return alloc->getCacheMemoryStats().cacheSize; }; - for (auto totalSize : totalSizes) { - auto dramCacheSize = getCacheSize(totalSize, 1); - for (auto n : numTiers) { - auto tieredCacheSize = getCacheSize(totalSize, n); - EXPECT_GT(dramCacheSize, tieredCacheSize); - EXPECT_GE(metaDataSize * n * 2, dramCacheSize - tieredCacheSize); + for (auto nTiers : numTiers) { + for (auto totalCacheSize : totalCacheSizes) { + if (totalCacheSize <= nTiers * Slab::kSize * 4) + continue; + LruAllocatorConfig cfg = + createTestCacheConfig(paths[nTiers], sizePairs[nTiers], + /* usePoisx */ true, totalCacheSize); + basicCheck(cfg, paths[nTiers], totalCacheSize); + testAddPool(cfg, 0, /* isSizeValid */ true, /* isTestMaxSize */ true); + testAddPool(cfg, 1); + testAddPool(cfg, totalCacheSize, /* isSizeValid */ false); + testAddPool(cfg, totalCacheSize / nTiers); } } }