First pass at adding Cachelib as Pluggable Secondary Cache

cachelib/adaptor/rocks_secondary_cache/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 3.4)
+
+find_package(cachelib REQUIRED)
+add_library(rocksdb_cachelib STATIC CachelibWrapper.cpp)
+target_link_libraries(rocksdb_cachelib PRIVATE cachelib)
+target_include_directories(rocksdb_cachelib PRIVATE ${CACHELIB_INCLUDE_DIR})
+
+# Suppresses errors from folly exceptions
+target_compile_options(rocksdb_cachelib PRIVATE -Wno-error=class-memaccess)
+set(cachelib_LIBS rocksdb_cachelib PARENT_SCOPE)
+set(cachelib_TARGETS rocksdb_cachelib PARENT_SCOPE)
+set(cachelib_FUNC register_CachelibObjects PARENT_SCOPE)

cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp

@@ -14,18 +14,17 @@
  * limitations under the License.
  */
 
-#include "cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h"
+#include "CachelibWrapper.h"
 
-#include "cachelib/facebook/utils/FbInternalRuntimeUpdateWrapper.h"
 #include "folly/init/Init.h"
 #include "folly/synchronization/Rcu.h"
 #include "rocksdb/version.h"
+#include "rocksdb/utilities/object_registry.h"
 
 namespace facebook {
 namespace rocks_secondary_cache {
 
 #define FB_CACHE_MAX_ITEM_SIZE 4 << 20
-using ApiWrapper = cachelib::FbInternalRuntimeUpdateWrapper<FbCache>;
 
 namespace {
 // We use a separate RCU domain since read side critical sections can block
@@ -131,6 +130,41 @@ class RocksCachelibWrapperHandle : public rocksdb::SecondaryCacheResultHandle {
 };
 } // namespace
 
+RockeCachelibWrapper::PrepareOptions(const ROCKSDB_NAMESPACE::ConfigOptions& opts) {
+  if (!cache_) {
+    std::unique_ptr<FbCache> cache;
+    cachelib::PoolId defaultPool;
+    FbCacheConfig config;
+    NvmCacheConfig nvmConfig;
+
+    nvmConfig.navyConfig.setBlockSize(options_.blockSize);
+    nvmConfig.navyConfig.setSimpleFile(options_.fileName,
+				       options_.size,
+				       /*truncateFile=*/true);
+    nvmConfig.navyConfig.blockCache().setRegionSize(options_.regionSize);
+    if (options_.admPolicy == "random") {
+      nvmConfig.navyConfig.enableRandomAdmPolicy().setAdmProbability(
+       options_.admProbability);
+    } else {
+      nvmConfig.navyConfig.enableDynamicRandomAdmPolicy()
+          .setMaxWriteRate(options_.maxWriteRate)
+          .setAdmWriteRate(options_.admissionWriteRate);
+    }
+    nvmConfig.enableFastNegativeLookups = true;
+
+    config.setCacheSize(options_.volatileSize)
+      .setCacheName(options_.cacheName)
+      .setAccessConfig(
+          {options_.bktPower /* bucket power */, options_.lockPower /* lock power */})
+      .enableNvmCache(nvmConfig)
+      .validate(); // will throw if bad config
+    cache_ = std::make_unique<FbCache>(config);
+    pool_ =
+      cache->addPool("default", cache_->getCacheMemoryStats().cacheSize);
+  }
+  return SecondaryCache::PreareOptions(opts);
+}
+
 RocksCachelibWrapper::~RocksCachelibWrapper() { Close(); }
 
 rocksdb::Status RocksCachelibWrapper::Insert(
@@ -225,26 +259,14 @@ void RocksCachelibWrapper::Close() {
     // sections already started to finish, and then delete the cache
     cache_.store(nullptr);
     GetRcuDomain().synchronize();
-    admin_.reset();
     delete cache;
   }
 }
 
-bool RocksCachelibWrapper::UpdateMaxWriteRateForDynamicRandom(
-    uint64_t maxRate) {
-  FbCache* cache = cache_.load();
-  bool ret = false;
-  if (cache) {
-    ret = ApiWrapper::updateMaxRateForDynamicRandomAP(*cache, maxRate);
-  }
-  return ret;
-}
-
 // Global cache object and a default cache pool
 std::unique_ptr<rocksdb::SecondaryCache> NewRocksCachelibWrapper(
     const RocksCachelibOptions& opts) {
   std::unique_ptr<FbCache> cache;
-  std::unique_ptr<cachelib::CacheAdmin> admin;
   cachelib::PoolId defaultPool;
   FbCacheConfig config;
   NvmCacheConfig nvmConfig;
@@ -274,15 +296,23 @@ std::unique_ptr<rocksdb::SecondaryCache> NewRocksCachelibWrapper(
   defaultPool =
       cache->addPool("default", cache->getCacheMemoryStats().cacheSize);
 
-  if (opts.fb303Stats) {
-    cachelib::CacheAdmin::Config adminConfig;
-    adminConfig.oncall = opts.oncallName;
-    admin = std::make_unique<cachelib::CacheAdmin>(*cache, adminConfig);
-  }
-
   return std::unique_ptr<rocksdb::SecondaryCache>(new RocksCachelibWrapper(
-      std::move(cache), std::move(admin), std::move(defaultPool)));
+      std::move(cache), std::move(defaultPool)));
 }
 
+#ifndef ROCKSDB_LITE
+int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&) {
+  library.AddFactory<ROCKSDB_NAMESPACE::SecondaryCache>(CachelibWrapper::kClassName(), 
+      [](const std::string& uri, std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache>* guard,
+         std::string* /*errmsg*/) {
+	RocksCachelibOptions options;
+	guard->reset(new CacheLibWrapper(options));
+        return guard->get();
+      });
+  return 1;
+}
+#endif // ROCKSDB_LITE
 } // namespace rocks_secondary_cache
 } // namespace facebook
+
+

cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h

@@ -16,11 +16,14 @@
 
 #pragma once
 #include "cachelib/allocator/CacheAllocator.h"
-#include "cachelib/facebook/admin/CacheAdmin.h"
 #include "rocksdb/secondary_cache.h"
 #include "rocksdb/types.h"
 #include "rocksdb/version.h"
 
+namespace ROCKSDB_NAMESPACE {
+class ObjectLibrary;
+} // namespace ROCKSDB_NAMESPACE
+
 namespace facebook {
 namespace rocks_secondary_cache {
 // Options structure for configuring a Cachelib SecondaryCache instance
@@ -82,37 +85,34 @@ using FbCacheReadHandle = typename FbCache::ReadHandle;
 using FbCacheItem = typename FbCache::Item;
 
 // The RocksCachelibWrapper is a concrete implementation of
-// rocksdb::SecondaryCache. It can be allocated using
+// ROCKSDB_NAMESPACE::SecondaryCache. It can be allocated using
 // NewRocksCachelibWrapper() and the resulting pointer
-// can be passed in rocksdb::LRUCacheOptions to
-// rocksdb::NewLRUCache().
+// can be passed in ROCKSDB_NAMESPACE::LRUCacheOptions to
+// ROCKSDB_NAMESPACE::NewLRUCache().
 //
 // Users can also cast a pointer to it and call methods on
 // it directly, especially custom methods that may be added
 // in the future.  For example -
-// std::unique_ptr<rocksdb::SecondaryCache> cache =
+// std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache> cache =
 //      NewRocksCachelibWrapper(opts);
 // static_cast<RocksCachelibWrapper*>(cache.get())->Erase(key);
-class RocksCachelibWrapper : public rocksdb::SecondaryCache {
+class RocksCachelibWrapper : public ROCKSDB_NAMESPACE::SecondaryCache {
  public:
-  RocksCachelibWrapper(std::unique_ptr<FbCache>&& cache,
-                       std::unique_ptr<cachelib::CacheAdmin>&& admin,
-                       cachelib::PoolId pool)
-      : cache_(std::move(cache).release()),
-        admin_(std::move(admin)),
-        pool_(pool) {}
+  RocksCachelibWrapper(const RocksCachelibOptions& options) :
+    options_(options) { }
   ~RocksCachelibWrapper() override;
 
-  const char* Name() const override { return "RocksCachelibWrapper"; }
+  static const char* kClassName() const { return "RocksCachelibWrapper"; }
+  const char* Name() const override { return kClassName(); }
 
-  rocksdb::Status Insert(
-      const rocksdb::Slice& key,
+  ROCKSDB_NAMESPACE::Status Insert(
+      const ROCKSDB_NAMESPACE::Slice& key,
       void* value,
-      const rocksdb::Cache::CacheItemHelper* helper) override;
+      const ROCKSDB_NAMESPACE::Cache::CacheItemHelper* helper) override;
 
-  std::unique_ptr<rocksdb::SecondaryCacheResultHandle> Lookup(
-      const rocksdb::Slice& key,
-      const rocksdb::Cache::CreateCallback& create_cb,
+  std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCacheResultHandle> Lookup(
+      const ROCKSDB_NAMESPACE::Slice& key,
+      const ROCKSDB_NAMESPACE::Cache::CreateCallback& create_cb,
       bool wait
 #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7)
       ,
@@ -125,10 +125,10 @@ class RocksCachelibWrapper : public rocksdb::SecondaryCache {
   bool SupportForceErase() const override { return false; }
 #endif
 
-  void Erase(const rocksdb::Slice& key) override;
+  void Erase(const ROCKSDB_NAMESPACE::Slice& key) override;
 
   void WaitAll(
-      std::vector<rocksdb::SecondaryCacheResultHandle*> handles) override;
+      std::vector<ROCKSDB_NAMESPACE::SecondaryCacheResultHandle*> handles) override;
 
   // TODO
   std::string GetPrintableOptions() const override { return ""; }
@@ -140,19 +140,22 @@ class RocksCachelibWrapper : public rocksdb::SecondaryCache {
   // ongoing lookups and inserts by other threads to be quiesced.
   void Close();
 
-  // If the admPolicy in RocksCachelibOptions was set to "dynamic_random",
-  // then this function can be called to update the max write rate for that
-  // policy.
-  bool UpdateMaxWriteRateForDynamicRandom(uint64_t maxRate);
-
+  ROCKSDB_NAMESPACE::Status PrepareOptions(const ConfigOptions& /*options*/) override;
+
  private:
+  RocksCachelibOptions options_;
   std::atomic<FbCache*> cache_;
-  std::unique_ptr<cachelib::CacheAdmin> admin_;
   cachelib::PoolId pool_;
 };
 
-// Allocate a new Cache instance with a rocksdb::TieredCache wrapper around it
-extern std::unique_ptr<rocksdb::SecondaryCache> NewRocksCachelibWrapper(
+// Allocate a new Cache instance with a ROCKSDB_NAMESPACE::TieredCache wrapper around it
+extern std::unique_ptr<ROCKSDB_NAMESPACE::SecondaryCache> NewRocksCachelibWrapper(
     const RocksCachelibOptions& opts);
+#ifndef ROCKSDB_LITE
+extern "C" {
+int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&);
+} // extern "C"
+
+#endif // ROCKSDB_LITE
 } // namespace rocks_secondary_cache
 } // namespace facebook