Introduce FileShmSegment for file-backed shared memory

It's implementation is mostly based on PosixShmSegment.

Also, extend ShmManager and ShmSegmentOpts to support this new
segment type.

Author: igchor

cachelib/allocator/CacheAllocator-inl.h

@@ -68,7 +68,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.accessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                              false, config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -79,7 +80,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.chainedItemAccessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                              false, config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -89,7 +91,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
       nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(),
                      config_.isNvmCacheTruncateAllocSizeEnabled()} {
   initCommon(false);
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+    PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -107,13 +110,15 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
       accessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.accessConfig,
-          shmManager_->attachShm(detail::kShmHashTableName),
+          shmManager_->attachShm(detail::kShmHashTableName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemAccessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.chainedItemAccessConfig,
-          shmManager_->attachShm(detail::kShmChainedItemHashTableName),
+          shmManager_->attachShm(detail::kShmChainedItemHashTableName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemLocks_(config_.chainedItemsLockPower,
@@ -130,7 +135,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
   // We will create a new info shm segment on shutDown(). If we don't remove
   // this info shm segment here and the new info shm segment's size is larger
   // than this one, creating new one will fail.
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+    PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -148,6 +154,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::createNewMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       getAllocatorConfig(config_),
       shmManager_
@@ -162,6 +169,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::restoreMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       deserializer_->deserialize<MemoryAllocator::SerializationType>(),
       shmManager_
@@ -265,7 +273,8 @@ void CacheAllocator<CacheTrait>::initWorkers() {
 
 template <typename CacheTrait>
 std::unique_ptr<Deserializer> CacheAllocator<CacheTrait>::createDeserializer() {
-  auto infoAddr = shmManager_->attachShm(detail::kShmInfoName);
+  auto infoAddr = shmManager_->attachShm(detail::kShmInfoName, nullptr,
+            ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm));
   return std::make_unique<Deserializer>(
       reinterpret_cast<uint8_t*>(infoAddr.addr),
       reinterpret_cast<uint8_t*>(infoAddr.addr) + infoAddr.size);
@@ -3041,8 +3050,11 @@ void CacheAllocator<CacheTrait>::saveRamCache() {
   std::unique_ptr<folly::IOBuf> ioBuf = serializedBuf.move();
   ioBuf->coalesce();
 
-  void* infoAddr =
-      shmManager_->createShm(detail::kShmInfoName, ioBuf->length()).addr;
+  ShmSegmentOpts opts;
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
+
+  void* infoAddr = shmManager_->createShm(detail::kShmInfoName, ioBuf->length(),
+      nullptr, opts).addr;
   Serializer serializer(reinterpret_cast<uint8_t*>(infoAddr),
                         reinterpret_cast<uint8_t*>(infoAddr) + ioBuf->length());
   serializer.writeToBuffer(std::move(ioBuf));
@@ -3386,7 +3398,7 @@ bool CacheAllocator<CacheTrait>::stopReaper(std::chrono::seconds timeout) {
 
 template <typename CacheTrait>
 bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
-    const std::string& cacheDir, bool posix) {
+    const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector<CacheMemoryTierConfig>& config */) {
   if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) {
     try {
       // cache dir exists. clean up only if there are no other processes
@@ -3405,6 +3417,12 @@ bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
     ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix);
     ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName,
                              posix);
+
+    // TODO(SHM_FILE): try to nuke segments of differente types (which require
+    // extra info)
+    // for (auto &tier : config) {
+    //   ShmManager::removeByName(cacheDir, tierShmName, config_.memoryTiers[i].opts);
+    // }
   }
   return true;
 }

cachelib/allocator/CacheAllocator.h

@@ -1035,7 +1035,8 @@ class CacheAllocator : public CacheBase {
   // returns true if there was no error in trying to cleanup the segment
   // because another process was attached. False if the user tried to clean up
   // and the cache was actually attached.
-  static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix);
+  static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix
+    /*TODO: const std::vector<CacheMemoryTierConfig>& config = {} */);
 
   // gives a relative offset to a pointer within the cache.
   uint64_t getItemPtrAsOffset(const void* ptr);

cachelib/allocator/TempShmMapping.cpp

@@ -34,7 +34,8 @@ TempShmMapping::TempShmMapping(size_t size)
 TempShmMapping::~TempShmMapping() {
   try {
     if (addr_) {
-      shmManager_->removeShm(detail::kTempShmCacheName.str());
+      shmManager_->removeShm(detail::kTempShmCacheName.str(),
+        PosixSysVSegmentOpts(false /* posix */));
     }
     if (shmManager_) {
       shmManager_.reset();
@@ -77,7 +78,8 @@ void* TempShmMapping::createShmMapping(ShmManager& shmManager,
     return shmAddr;
   } catch (...) {
     if (shmAddr) {
-      shmManager.removeShm(detail::kTempShmCacheName.str());
+      shmManager.removeShm(detail::kTempShmCacheName.str(),
+        PosixSysVSegmentOpts(false /* posix */));
     } else {
       munmap(addr, size);
     }

cachelib/shm/CMakeLists.txt

@@ -16,6 +16,7 @@ add_thrift_file(SHM shm.thrift frozen2)
 
 add_library (cachelib_shm
   ${SHM_THRIFT_FILES}
+  FileShmSegment.cpp
   PosixShmSegment.cpp
   ShmCommon.cpp
   ShmManager.cpp