- Add memory tier configs to cache allocator based on NUMA bindings

- Add in MemoryTier tests
- Increase max number of tiers to 2 for tiers tests

cachelib/allocator/CMakeLists.txt

@@ -117,6 +117,8 @@ if (BUILD_TESTS)
   add_test (tests/ChainedHashTest.cpp)
   add_test (tests/AllocatorResizeTypeTest.cpp)
   add_test (tests/AllocatorHitStatsTypeTest.cpp)
+  add_test (tests/AllocatorMemoryTiersTest.cpp)
+  add_test (tests/MemoryTiersTest.cpp)
   add_test (tests/MultiAllocatorTest.cpp)
   add_test (tests/NvmAdmissionPolicyTest.cpp)
   add_test (tests/CacheAllocatorConfigTest.cpp)

cachelib/allocator/CacheAllocator-inl.h

@@ -53,6 +53,7 @@ CacheAllocator<CacheTrait>::CacheAllocator(
     : isOnShm_{type != InitMemType::kNone ? true
                                           : config.memMonitoringEnabled()},
       config_(config.validate()),
+      memoryTierConfigs(config.getMemoryTierConfigs()),
       tempShm_(type == InitMemType::kNone && isOnShm_
                    ? std::make_unique<TempShmMapping>(config_.getCacheSize())
                    : nullptr),
@@ -110,9 +111,10 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts() {
   opts.alignment = sizeof(Slab);
   auto memoryTierConfigs = config_.getMemoryTierConfigs();
   // TODO: we support single tier so far
-  XDCHECK_EQ(memoryTierConfigs.size(), 1ul);
+  if (memoryTierConfigs.size() > 1) {
+    throw std::invalid_argument("CacheLib only supports a single memory tier");
+  }
   opts.memBindNumaNodes = memoryTierConfigs[0].getMemBind();
-
   return opts;
 }
 

cachelib/allocator/CacheAllocator.h

@@ -1159,6 +1159,9 @@ class CacheAllocator : public CacheBase {
   // whether it is object-cache
   bool isObjectCache() const override final { return false; }
 
+  // combined pool size for all memory tiers
+  size_t getPoolSize(PoolId pid) const;
+
   // pool stats by pool id
   PoolStats getPoolStats(PoolId pid) const override final;
 
@@ -1959,6 +1962,8 @@ class CacheAllocator : public CacheBase {
 
   Config config_{};
 
+  const typename Config::MemoryTierConfigs memoryTierConfigs;
+
   // Manages the temporary shared memory segment for memory allocator that
   // is not persisted when cache process exits.
   std::unique_ptr<TempShmMapping> tempShm_;

cachelib/allocator/CacheAllocatorConfig.h

@@ -207,6 +207,9 @@ class CacheAllocatorConfig {
   // Accepts vector of MemoryTierCacheConfig. Each vector element describes
   // configuration for a single memory cache tier. Tier sizes are specified as
   // ratios, the number of parts of total cache size each tier would occupy.
+  // @throw std::invalid_argument if:
+  // - the size of configs is 0
+  // - the size of configs is greater than kMaxCacheMemoryTiers
   CacheAllocatorConfig& configureMemoryTiers(const MemoryTierConfigs& configs);
 
   // Return reference to MemoryTierCacheConfigs.
@@ -374,8 +377,7 @@ class CacheAllocatorConfig {
   std::map<std::string, std::string> serialize() const;
 
   // The max number of memory cache tiers
-  // TODO: increase this number when multi-tier configs are enabled
-  inline static const size_t kMaxCacheMemoryTiers = 1;
+  inline static const size_t kMaxCacheMemoryTiers = 2;
 
   // Cache name for users to indentify their own cache.
   std::string cacheName{""};

cachelib/allocator/MemoryTierCacheConfig.h

@@ -23,10 +23,7 @@ namespace cachelib {
 class MemoryTierCacheConfig {
  public:
   // Creates instance of MemoryTierCacheConfig for Posix/SysV Shared memory.
-  static MemoryTierCacheConfig fromShm() {
-    // TODO: expand this method when adding support for file-mapped memory
-    return MemoryTierCacheConfig();
-  }
+  static MemoryTierCacheConfig fromShm() { return MemoryTierCacheConfig(); }
 
   // Specifies ratio of this memory tier to other tiers. Absolute size
   // of each tier can be calculated as:
@@ -49,7 +46,7 @@ class MemoryTierCacheConfig {
 
   const NumaBitMask& getMemBind() const noexcept { return numaNodes; }
 
-  size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) {
+  size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
     // TODO: Call this method when tiers are enabled in allocator
     // to calculate tier sizes in bytes.
     if (!partitionNum) {

cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) Intel Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/allocator/tests/AllocatorMemoryTiersTest.h"
+
+namespace facebook {
+namespace cachelib {
+namespace tests {
+
+using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest<LruAllocator>;
+
+// TODO(MEMORY_TIER): add more tests with different eviction policies
+TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid1) {
+  this->testMultiTiersValid1();
+}
+
+} // end of namespace tests
+} // end of namespace cachelib
+} // end of namespace facebook

cachelib/allocator/tests/AllocatorMemoryTiersTest.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "cachelib/allocator/CacheAllocatorConfig.h"
+#include "cachelib/allocator/MemoryTierCacheConfig.h"
+#include "cachelib/allocator/tests/TestBase.h"
+
+namespace facebook {
+namespace cachelib {
+namespace tests {
+
+template <typename AllocatorT>
+class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
+ public:
+  void testMultiTiersValid1() {
+    typename AllocatorT::Config config;
+    config.setCacheSize(100 * Slab::kSize);
+    ASSERT_NO_THROW(config.configureMemoryTiers(
+        {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
+             std::string("0")),
+         MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
+             std::string("0"))}));
+  }
+};
+} // namespace tests
+} // namespace cachelib
+} // namespace facebook

cachelib/allocator/tests/AllocatorTypeTest.cpp

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#include "cachelib/allocator/MemoryTierCacheConfig.h"
 #include "cachelib/allocator/tests/BaseAllocatorTest.h"
 #include "cachelib/allocator/tests/TestBase.h"
 
@@ -226,6 +227,11 @@ TYPED_TEST(BaseAllocatorTest, ReaperSkippingSlabTraversalWhileSlabReleasing) {
 }
 
 TYPED_TEST(BaseAllocatorTest, ReaperShutDown) { this->testReaperShutDown(); }
+TYPED_TEST(BaseAllocatorTest, ReaperShutDownFile) {
+  this->testReaperShutDown(
+      {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
+          std::string("0"))});
+}
 
 TYPED_TEST(BaseAllocatorTest, ShutDownWithActiveHandles) {
   this->testShutDownWithActiveHandles();

cachelib/allocator/tests/BaseAllocatorTest.h

@@ -1253,7 +1253,8 @@ class BaseAllocatorTest : public AllocatorTest<AllocatorT> {
     this->testLruLength(alloc, poolId, sizes, keyLen, evictedKeys);
   }
 
-  void testReaperShutDown() {
+  void testReaperShutDown(
+      typename AllocatorT::Config::MemoryTierConfigs cfgs = {}) {
     const size_t nSlabs = 20;
     const size_t size = nSlabs * Slab::kSize;
 
@@ -1263,6 +1264,8 @@ class BaseAllocatorTest : public AllocatorTest<AllocatorT> {
     config.setAccessConfig({8, 8});
     config.enableCachePersistence(this->cacheDir_);
     config.enableItemReaperInBackground(std::chrono::seconds(1), {});
+    if (cfgs.size())
+      config.configureMemoryTiers(cfgs);
     std::vector<typename AllocatorT::Key> keys;
     {
       AllocatorT alloc(AllocatorT::SharedMemNew, config);

cachelib/allocator/tests/MemoryTiersTest.cpp

@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/Random.h>
+
+#include <numeric>
+
+#include "cachelib/allocator/CacheAllocator.h"
+#include "cachelib/allocator/tests/TestBase.h"
+
+namespace facebook {
+namespace cachelib {
+namespace tests {
+
+using LruAllocatorConfig = CacheAllocatorConfig<LruAllocator>;
+using LruMemoryTierConfigs = LruAllocatorConfig::MemoryTierConfigs;
+using Strings = std::vector<std::string>;
+using Ratios = std::vector<size_t>;
+
+constexpr size_t MB = 1024ULL * 1024ULL;
+constexpr size_t GB = MB * 1024ULL;
+
+const size_t defaultTotalCacheSize{1 * GB};
+const std::string defaultCacheDir{"/tmp/metadataDir"};
+
+template <typename Allocator>
+class MemoryTiersTest : public AllocatorTest<Allocator> {
+ public:
+  void basicCheck(LruAllocatorConfig& actualConfig,
+                  size_t expectedTotalCacheSize = defaultTotalCacheSize,
+                  const std::string& expectedCacheDir = defaultCacheDir) {
+    EXPECT_EQ(actualConfig.getCacheSize(), expectedTotalCacheSize);
+    auto configs = actualConfig.getMemoryTierConfigs();
+
+    size_t sum_ratios = std::accumulate(
+        configs.begin(), configs.end(), 0UL,
+        [](const size_t i, const MemoryTierCacheConfig& config) {
+          return i + config.getRatio();
+        });
+    size_t sum_sizes = std::accumulate(
+        configs.begin(), configs.end(), 0UL,
+        [&](const size_t i, const MemoryTierCacheConfig& config) {
+          return i + config.calculateTierSize(actualConfig.getCacheSize(),
+                                              sum_ratios);
+        });
+
+    EXPECT_GE(expectedTotalCacheSize, sum_ratios * Slab::kSize);
+    EXPECT_LE(sum_sizes, expectedTotalCacheSize);
+    EXPECT_GE(sum_sizes, expectedTotalCacheSize - configs.size() * Slab::kSize);
+  }
+
+  LruAllocatorConfig createTestCacheConfig(
+      const Ratios& tierRatios = {1},
+      bool setPosixForShm = true,
+      size_t cacheSize = defaultTotalCacheSize,
+      const std::string& cacheDir = defaultCacheDir) {
+    LruAllocatorConfig cfg;
+    cfg.setCacheSize(cacheSize).enableCachePersistence(cacheDir);
+
+    if (setPosixForShm)
+      cfg.usePosixForShm();
+    LruMemoryTierConfigs tierConfigs;
+    tierConfigs.reserve(tierRatios.size());
+    for (auto i = 0; i < tierRatios.size(); ++i) {
+      tierConfigs.push_back(MemoryTierCacheConfig::fromShm()
+                                .setRatio(tierRatios[i])
+                                .setMemBind(std::string("0")));
+    }
+
+    cfg.configureMemoryTiers(tierConfigs);
+    return cfg;
+  }
+
+  LruAllocatorConfig createTieredCacheConfig(size_t totalCacheSize,
+                                             size_t numTiers = 2) {
+    LruAllocatorConfig tieredCacheConfig{};
+    std::vector<MemoryTierCacheConfig> configs;
+    for (auto i = 1; i <= numTiers; ++i) {
+      configs.push_back(MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind(
+          std::string("0")));
+    }
+    tieredCacheConfig.setCacheSize(totalCacheSize)
+        .enableCachePersistence(
+            folly::sformat("/tmp/multi-tier-test/{}", ::getpid()))
+        .usePosixForShm()
+        .configureMemoryTiers(configs);
+    return tieredCacheConfig;
+  }
+
+  LruAllocatorConfig createDramCacheConfig(size_t totalCacheSize) {
+    LruAllocatorConfig dramConfig{};
+    dramConfig.setCacheSize(totalCacheSize);
+    return dramConfig;
+  }
+
+  void validatePoolSize(PoolId poolId,
+                        std::unique_ptr<LruAllocator>& allocator,
+                        size_t expectedSize) {
+    size_t actualSize = allocator->getPool(poolId).getPoolSize();
+    EXPECT_EQ(actualSize, expectedSize);
+  }
+
+  void testAddPool(std::unique_ptr<LruAllocator>& alloc,
+                   size_t poolSize,
+                   bool isSizeValid = true,
+                   size_t numTiers = 2) {
+    if (isSizeValid) {
+      auto pool = alloc->addPool("validPoolSize", poolSize);
+      EXPECT_LE(alloc->getPool(pool).getPoolSize(), poolSize);
+      if (poolSize >= numTiers * Slab::kSize)
+        EXPECT_GE(alloc->getPool(pool).getPoolSize(),
+                  poolSize - numTiers * Slab::kSize);
+    } else {
+      EXPECT_THROW(alloc->addPool("invalidPoolSize", poolSize),
+                   std::invalid_argument);
+      // TODO: test this for all tiers
+      EXPECT_EQ(alloc->getPoolIds().size(), 0);
+    }
+  }
+};
+
+using LruMemoryTiersTest = MemoryTiersTest<LruAllocator>;
+
+TEST_F(LruMemoryTiersTest, TestValid1TierConfig) {
+  LruAllocatorConfig cfg = createTestCacheConfig().validate();
+  basicCheck(cfg);
+}
+
+TEST_F(LruMemoryTiersTest, TestValid2TierConfig) {
+  LruAllocatorConfig cfg = createTestCacheConfig({1, 1});
+  basicCheck(cfg);
+}
+
+TEST_F(LruMemoryTiersTest, TestValid2TierRatioConfig) {
+  LruAllocatorConfig cfg = createTestCacheConfig({5, 2});
+  basicCheck(cfg);
+}
+
+TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigNumberOfPartitionsTooLarge) {
+  EXPECT_THROW(createTestCacheConfig({defaultTotalCacheSize, 1}).validate(),
+               std::invalid_argument);
+}
+
+TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesAndRatioNotSet) {
+  EXPECT_THROW(createTestCacheConfig({1, 0}), std::invalid_argument);
+}
+
+TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatiosCacheSizeNotSet) {
+  EXPECT_THROW(createTestCacheConfig({1, 1}, true,
+                                     /* cacheSize */ 0)
+                   .validate(),
+               std::invalid_argument);
+}
+
+TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatioNotSet) {
+  EXPECT_THROW(createTestCacheConfig({1, 0}), std::invalid_argument);
+}
+
+TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) {
+  EXPECT_THROW(createTestCacheConfig({0, 0}), std::invalid_argument);
+}
+} // namespace tests
+} // namespace cachelib
+} // namespace facebook