Skip to content

Commit a4c4ab3

Browse files
vinser52vinser52
authored
Merge pull request #13 from vinser52/numa_binding
Initial support of NUMA bindings
2 parents 63c9029 + b0e2574 commit a4c4ab3

15 files changed

+299
-6
lines changed

‎cachelib/allocator/CacheAllocator-inl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
125125
ShmSegmentOpts opts;
126126
opts.alignment = sizeof(Slab);
127127
opts.typeOpts = memoryTierConfigs[tid].getShmTypeOpts();
128+
opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind();
128129
if (auto *v = std::get_if<PosixSysVSegmentOpts>(&opts.typeOpts)) {
129130
v->usePosix = config_.usePosixShm;
130131
}

‎cachelib/allocator/MemoryTierCacheConfig.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,16 @@ class MemoryTierCacheConfig {
5353

5454
size_t getRatio() const noexcept { return ratio; }
5555

56+
// Allocate memory only from specified NUMA nodes
57+
MemoryTierCacheConfig& setMemBind(const std::vector<size_t>& _numaNodes) {
58+
numaNodes = _numaNodes;
59+
return *this;
60+
}
61+
62+
std::vector<size_t> getMemBind() const {
63+
return numaNodes;
64+
}
65+
5666
size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
5767
// TODO: Call this method when tiers are enabled in allocator
5868
// to calculate tier sizes in bytes.
@@ -82,6 +92,9 @@ class MemoryTierCacheConfig {
8292
// Options specific to shm type
8393
ShmTypeOpts shmOpts;
8494

95+
// Numa node(s) to bind the tier
96+
std::vector<size_t> numaNodes;
97+
8598
MemoryTierCacheConfig() = default;
8699
};
87100
} // namespace cachelib

‎cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@ namespace tests {
2323
using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest<LruAllocator>;
2424

2525
// TODO(MEMORY_TIER): add more tests with different eviction policies
26-
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); }
27-
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); }
26+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileInvalid) { this->testMultiTiersFormFileInvalid(); }
27+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileValid) { this->testMultiTiersFromFileValid(); }
2828
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); }
29+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsSysVValid) { this->testMultiTiersNumaBindingsSysVValid(); }
30+
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsPosixValid) { this->testMultiTiersNumaBindingsPosixValid(); }
2931

3032
} // end of namespace tests
3133
} // end of namespace cachelib

‎cachelib/allocator/tests/AllocatorMemoryTiersTest.h‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ namespace tests {
2727
template <typename AllocatorT>
2828
class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
2929
public:
30-
void testMultiTiersInvalid() {
30+
void testMultiTiersFormFileInvalid() {
3131
typename AllocatorT::Config config;
3232
config.setCacheSize(100 * Slab::kSize);
3333
config.configureMemoryTiers({
@@ -42,7 +42,7 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
4242
std::invalid_argument);
4343
}
4444

45-
void testMultiTiersValid() {
45+
void testMultiTiersFromFileValid() {
4646
typename AllocatorT::Config config;
4747
config.setCacheSize(100 * Slab::kSize);
4848
config.enableCachePersistence("/tmp");
@@ -83,6 +83,47 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
8383
ASSERT(handle != nullptr);
8484
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
8585
}
86+
87+
void testMultiTiersNumaBindingsSysVValid() {
88+
typename AllocatorT::Config config;
89+
config.setCacheSize(100 * Slab::kSize);
90+
config.enableCachePersistence("/tmp");
91+
config.configureMemoryTiers({
92+
MemoryTierCacheConfig::fromShm()
93+
.setRatio(1).setMemBind({0}),
94+
MemoryTierCacheConfig::fromShm()
95+
.setRatio(1).setMemBind({0})
96+
});
97+
98+
auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
99+
ASSERT(alloc != nullptr);
100+
101+
auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize);
102+
auto handle = alloc->allocate(pool, "key", std::string("value").size());
103+
ASSERT(handle != nullptr);
104+
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
105+
}
106+
107+
void testMultiTiersNumaBindingsPosixValid() {
108+
typename AllocatorT::Config config;
109+
config.setCacheSize(100 * Slab::kSize);
110+
config.enableCachePersistence("/tmp");
111+
config.usePosixForShm();
112+
config.configureMemoryTiers({
113+
MemoryTierCacheConfig::fromShm()
114+
.setRatio(1).setMemBind({0}),
115+
MemoryTierCacheConfig::fromShm()
116+
.setRatio(1).setMemBind({0})
117+
});
118+
119+
auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
120+
ASSERT(alloc != nullptr);
121+
122+
auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize);
123+
auto handle = alloc->allocate(pool, "key", std::string("value").size());
124+
ASSERT(handle != nullptr);
125+
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
126+
}
86127
};
87128
} // namespace tests
88129
} // namespace cachelib

‎cachelib/cachebench/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,5 +89,6 @@ if (BUILD_TESTS)
8989
add_test (consistency/tests/ValueHistoryTest.cpp)
9090
add_test (consistency/tests/ValueTrackerTest.cpp)
9191
add_test (util/tests/NandWritesTest.cpp)
92+
add_test (util/tests/MemoryTierConfigTest.cpp)
9293
add_test (cache/tests/TimeStampTickerTest.cpp)
9394
endif()

‎cachelib/cachebench/util/CacheConfig.cpp‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,53 @@ std::shared_ptr<RebalanceStrategy> CacheConfig::getRebalanceStrategy() const {
137137
MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) {
138138
JSONSetVal(configJson, file);
139139
JSONSetVal(configJson, ratio);
140+
JSONSetVal(configJson, memBindNodes);
140141

141-
checkCorrectSize<MemoryTierConfig, 40>();
142+
checkCorrectSize<MemoryTierConfig, 72>();
143+
}
144+
145+
static bool starts_with() {return true;}
146+
147+
std::vector<size_t> MemoryTierConfig::parseNumaNodes() {
148+
std::vector<size_t> numaNodes;
149+
150+
std::vector<folly::StringPiece> tokens;
151+
folly::split(",", memBindNodes, tokens, true /*ignore empty*/);
152+
for(const auto &token : tokens) {
153+
if(token.startsWith("!")) {
154+
throw std::invalid_argument(folly::sformat(
155+
"invalid NUMA nodes binding in memory tier config: {} "
156+
"inverse !N or !N-N is not supported "
157+
"nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
158+
token));
159+
}
160+
else if(token.startsWith("+")) {
161+
throw std::invalid_argument(folly::sformat(
162+
"invalid NUMA nodes binding in memory tier config: {} "
163+
"relative nodes are not supported. "
164+
"nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
165+
token));
166+
}
167+
else if (token.contains("-")) {
168+
size_t begin, end;
169+
if(folly::split("-", token, begin, end) && begin < end) {
170+
while(begin <=end) {
171+
numaNodes.push_back(begin++);
172+
}
173+
} else {
174+
throw std::invalid_argument(folly::sformat(
175+
"invalid NUMA nodes binding in memory tier config: {} "
176+
"Invalid range format. "
177+
"nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
178+
token));
179+
}
180+
}
181+
else {
182+
numaNodes.push_back(folly::to<size_t>(token));
183+
}
184+
}
185+
186+
return numaNodes;
142187
}
143188

144189
} // namespace cachebench

‎cachelib/cachebench/util/CacheConfig.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,13 @@ struct MemoryTierConfig : public JSONConfig {
4848
MemoryTierCacheConfig getMemoryTierCacheConfig() {
4949
MemoryTierCacheConfig config = memoryTierCacheConfigFromSource();
5050
config.setRatio(ratio);
51+
config.setMemBind(parseNumaNodes());
5152
return config;
5253
}
5354

5455
std::string file{""};
5556
size_t ratio{0};
57+
std::string memBindNodes{""};
5658

5759
private:
5860
MemoryTierCacheConfig memoryTierCacheConfigFromSource() {
@@ -62,6 +64,8 @@ struct MemoryTierConfig : public JSONConfig {
6264
return MemoryTierCacheConfig::fromFile(file);
6365
}
6466
}
67+
68+
std::vector<size_t> parseNumaNodes();
6569
};
6670

6771
struct CacheConfig : public JSONConfig {

‎cachelib/cachebench/util/tests/MemoryTierConfigTest.cpp‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright (c) Facebook, Inc. and its affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// Copyright 2022-present Facebook. All Rights Reserved.
18+
19+
#include <algorithm>
20+
21+
#include <gmock/gmock.h>
22+
#include <gtest/gtest.h>
23+
24+
#include "cachelib/cachebench/util/CacheConfig.h"
25+
26+
namespace facebook {
27+
namespace cachelib {
28+
namespace cachebench {
29+
30+
TEST(MemoryTierConfigTest, MemBind_SingleNumaNode) {
31+
const std::string configString =
32+
"{"
33+
" \"ratio\": 1,"
34+
" \"memBindNodes\": 1"
35+
"}";
36+
37+
const std::vector<size_t> expectedNumaNodes = {1};
38+
39+
auto configJson = folly::parseJson(folly::json::stripComments(configString));
40+
41+
MemoryTierConfig memoryTierConfig(configJson);
42+
MemoryTierCacheConfig tierCacheConfig = memoryTierConfig.getMemoryTierCacheConfig();
43+
44+
auto parsedNumaNodes = tierCacheConfig.getMemBind();
45+
ASSERT_TRUE(std::equal(expectedNumaNodes.begin(), expectedNumaNodes.end(), parsedNumaNodes.begin()));
46+
}
47+
48+
TEST(MemoryTierConfigTest, MemBind_RangeNumaNodes) {
49+
const std::string configString =
50+
"{"
51+
" \"ratio\": 1,"
52+
" \"memBindNodes\": \"0-2\""
53+
"}";
54+
55+
const std::vector<size_t> expectedNumaNodes = {0, 1, 2};
56+
57+
auto configJson = folly::parseJson(folly::json::stripComments(configString));
58+
59+
MemoryTierConfig memoryTierConfig(configJson);
60+
MemoryTierCacheConfig tierCacheConfig = memoryTierConfig.getMemoryTierCacheConfig();
61+
62+
auto parsedNumaNodes = tierCacheConfig.getMemBind();
63+
ASSERT_TRUE(std::equal(expectedNumaNodes.begin(), expectedNumaNodes.end(), parsedNumaNodes.begin()));
64+
}
65+
66+
TEST(MemoryTierConfigTest, MemBind_SingleAndRangeNumaNodes) {
67+
const std::string configString =
68+
"{"
69+
" \"ratio\": 1,"
70+
" \"memBindNodes\": \"0,2-5\""
71+
"}";
72+
73+
const std::vector<size_t> expectedNumaNodes = {0, 2, 3, 4, 5};
74+
75+
auto configJson = folly::parseJson(folly::json::stripComments(configString));
76+
77+
MemoryTierConfig memoryTierConfig(configJson);
78+
MemoryTierCacheConfig tierCacheConfig = memoryTierConfig.getMemoryTierCacheConfig();
79+
80+
auto parsedNumaNodes = tierCacheConfig.getMemBind();
81+
ASSERT_TRUE(std::equal(expectedNumaNodes.begin(), expectedNumaNodes.end(), parsedNumaNodes.begin()));
82+
}
83+
84+
} // namespace facebook
85+
} // namespace cachelib
86+
} // namespace cachebench

‎cachelib/shm/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ add_library (cachelib_shm
2525
add_dependencies(cachelib_shm thrift_generated_files)
2626
target_link_libraries(cachelib_shm PUBLIC
2727
cachelib_common
28+
numa
2829
)
2930

3031
install(TARGETS cachelib_shm

‎cachelib/shm/PosixShmSegment.cpp‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include <sys/mman.h>
2222
#include <sys/stat.h>
2323
#include <sys/types.h>
24+
#include <numa.h>
25+
#include <numaif.h>
2426

2527
#include "cachelib/common/Utils.h"
2628

@@ -176,13 +178,52 @@ void* PosixShmSegment::mapAddress(void* addr) const {
176178
util::throwSystemError(EINVAL, "Address already mapped");
177179
}
178180
XDCHECK(retAddr == addr || addr == nullptr);
181+
memBind(addr);
179182
return retAddr;
180183
}
181184

182185
void PosixShmSegment::unMap(void* addr) const {
183186
detail::munmapImpl(addr, getSize());
184187
}
185188

189+
static void forcePageAllocation(void* addr, size_t size, size_t pageSize) {
190+
for(volatile char* curAddr = (char*)addr; curAddr < (char*)addr+size; curAddr += pageSize) {
191+
*curAddr = *curAddr;
192+
}
193+
}
194+
195+
void PosixShmSegment::memBind(void* addr) const {
196+
if(opts_.memBindNumaNodes.empty()) return;
197+
198+
struct bitmask *oldNodeMask = numa_allocate_nodemask();
199+
int oldMode = 0;
200+
struct bitmask *nodesMask = numa_allocate_nodemask();
201+
auto guard = folly::makeGuard([&] { numa_bitmask_free(nodesMask); numa_bitmask_free(oldNodeMask); });
202+
203+
for(auto node : opts_.memBindNumaNodes) {
204+
numa_bitmask_setbit(nodesMask, node);
205+
}
206+
207+
// mbind() cannot be used because mmap was called with MAP_SHARED flag
208+
// But we can set memory policy for current thread and force page allcoation.
209+
// The following logic is used:
210+
// 1. Remember current memory policy for the current thread
211+
// 2. Set new memory policy as specifiec by config
212+
// 3. Force page allocation by touching every page in the segment
213+
// 4. Restore memory policy
214+
215+
// Remember current memory policy
216+
get_mempolicy(&oldMode, oldNodeMask->maskp, oldNodeMask->size, nullptr, 0);
217+
218+
// Set memory bindings
219+
set_mempolicy(MPOL_BIND, nodesMask->maskp, nodesMask->size);
220+
221+
forcePageAllocation(addr, getSize(), detail::getPageSize(opts_.pageSize));
222+
223+
// Restore memory policy for the thread
224+
set_mempolicy(oldMode, nodesMask->maskp, nodesMask->size);
225+
}
226+
186227
std::string PosixShmSegment::createKeyForName(
187228
const std::string& name) noexcept {
188229
// ensure that the slash is always there in the head. repetitive

0 commit comments

Comments
 (0)