Implement NUMA binding support for SysVShmSegment

Author: vinser52

cachelib/allocator/CacheAllocator-inl.h

@@ -125,6 +125,7 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
   opts.typeOpts = memoryTierConfigs[tid].getShmTypeOpts();
+  opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind();
   if (auto *v = std::get_if<PosixSysVSegmentOpts>(&opts.typeOpts)) {
     v->usePosix = config_.usePosixShm;
   }

cachelib/allocator/MemoryTierCacheConfig.h

@@ -53,6 +53,16 @@ class MemoryTierCacheConfig {
 
   size_t getRatio() const noexcept { return ratio; }
 
+  // Allocate memory only from specified NUMA nodes
+  MemoryTierCacheConfig& setMemBind(const std::vector<size_t>& _numaNodes) {
+    numaNodes = _numaNodes;
+    return *this;
+  }
+
+  std::vector<size_t> getMemBind() const {
+    return numaNodes;
+  }
+
   size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
     // TODO: Call this method when tiers are enabled in allocator
     // to calculate tier sizes in bytes.
@@ -82,6 +92,9 @@ class MemoryTierCacheConfig {
   // Options specific to shm type
   ShmTypeOpts shmOpts;
 
+  // Numa node(s) to bind the tier
+  std::vector<size_t> numaNodes;
+
   MemoryTierCacheConfig() = default;
 };
 } // namespace cachelib

cachelib/shm/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library (cachelib_shm
 add_dependencies(cachelib_shm thrift_generated_files)
 target_link_libraries(cachelib_shm PUBLIC
   cachelib_common
+  numa
 )
 
 install(TARGETS cachelib_shm

cachelib/shm/ShmCommon.h

@@ -93,6 +93,7 @@ struct ShmSegmentOpts {
   PageSizeT pageSize{PageSizeT::NORMAL};
   bool readOnly{false};
   size_t alignment{1}; // alignment for mapping.
+  std::vector<size_t> memBindNumaNodes;
   // opts specific to segment type
   ShmTypeOpts typeOpts{PosixSysVSegmentOpts(false)};
 

cachelib/shm/SysVShmSegment.cpp

@@ -18,8 +18,11 @@
 
 #include <folly/hash/Hash.h>
 #include <folly/logging/xlog.h>
+#include <folly/ScopeGuard.h>
 #include <sys/mman.h>
 #include <sys/shm.h>
+#include <numa.h>
+#include <numaif.h>
 
 #include "cachelib/common/Utils.h"
 
@@ -184,6 +187,50 @@ void shmCtlImpl(int shmid, int cmd, shmid_ds* buf) {
   }
 }
 
+void mbindImpl(void *addr, unsigned long len, int mode,
+               const std::vector<size_t>& memBindNumaNodes,
+               unsigned int flags) {
+  struct bitmask *nodesMask = numa_allocate_nodemask();
+  auto guard = folly::makeGuard([&] { numa_bitmask_free(nodesMask); });
+
+  for(auto node : memBindNumaNodes) {
+    numa_bitmask_setbit(nodesMask, node);
+  }              
+  
+  long ret = mbind(addr, len, mode, nodesMask->maskp, nodesMask->size, flags);
+  if(ret == 0) return;
+
+  switch (errno) {
+  case EFAULT:
+    util::throwSystemError(errno);
+    break;
+  case EINVAL:
+    util::throwSystemError(errno, "Invalid parameters when bind segment to NUMA node(s)");
+    break;
+  case EIO:
+    if(flags & MPOL_MF_STRICT) {
+      util::throwSystemError(errno, "Segment already allocated on another NUMA node that does not follow the policy.");
+    }
+    if(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL )) {
+      util::throwSystemError(errno, "Segment already allocated but kernel was unable to move it to specified NUMA node(s).");
+    }
+    util::throwSystemError(errno, "Invalid errno");
+    break;
+  case ENOMEM:
+    util::throwSystemError(errno, "Could not bind memory. Insufficient kernel memory was available");
+    break;
+  case EPERM:
+    if(flags & MPOL_MF_MOVE_ALL) {
+      util::throwSystemError(errno, "Process does not have the CAP_SYS_NICE privilege to bind segment with MPOL_MF_MOVE_ALL flag");
+    }
+    util::throwSystemError(errno, "Invalid errno");
+    break;
+  default:
+    XDCHECK(false);
+    util::throwSystemError(errno, "Invalid errno");
+  }
+}
+
 } // namespace detail
 
 void ensureSizeforHugePage(size_t size) {
@@ -270,11 +317,17 @@ void* SysVShmSegment::mapAddress(void* addr) const {
 
   void* retAddr = detail::shmAttachImpl(shmid_, addr, shmFlags);
   XDCHECK(retAddr == addr || addr == nullptr);
+  memBind(retAddr);
   return retAddr;
 }
 
 void SysVShmSegment::unMap(void* addr) const { detail::shmDtImpl(addr); }
 
+void SysVShmSegment::memBind(void* addr) const {
+  if(opts_.memBindNumaNodes.empty()) return;
+  detail::mbindImpl(addr, getSize(), MPOL_BIND, opts_.memBindNumaNodes, 0);
+}
+
 void SysVShmSegment::markForRemoval() {
   if (isMarkedForRemoval()) {
     return;

cachelib/shm/SysVShmSegment.h

@@ -100,6 +100,7 @@ class SysVShmSegment : public ShmBase {
   void lockPagesInMemory() const;
   void createReferenceMapping();
   void deleteReferenceMapping() const;
+  void memBind(void* addr) const;
 
   //  the key identifier for the shared memory
   KeyType key_{kInvalidKey};

contrib/prerequisites-centos8.sh

@@ -57,7 +57,8 @@ sudo dnf --enablerepo="$POWERTOOLS_REPO" install -y \
   libsodium-static \
   libdwarf-static \
   boost-static \
-  double-conversion-static
+  double-conversion-static \
+  numactl-devel
 
 #Do not install these from OS packages - they are typically outdated.
 #gflags-devel \