Use individual thread pool for background GC (#126) (#134)

* Use individual thread pool for background GC (#126)
Signed-off-by: Connor1996 <zbk602423539@gmail.com>

CMakeLists.txt

@@ -9,7 +9,7 @@ if (NOT ROCKSDB_GIT_REPO)
 endif()
 
 if (NOT ROCKSDB_GIT_BRANCH)
-  set(ROCKSDB_GIT_BRANCH "tikv-3.0")
+  set(ROCKSDB_GIT_BRANCH "tikv-3.x")
 endif()
 
 if (NOT DEFINED ROCKSDB_DIR)

src/db_impl.cc

@@ -5,10 +5,10 @@
 #endif
 
 #include <inttypes.h>
-
 #include "port/port.h"
 #include "util/autovector.h"
 #include "util/log_buffer.h"
+#include "util/threadpool_imp.h"
 
 #include "base_db_listener.h"
 #include "blob_file_builder.h"
@@ -254,8 +254,12 @@ Status TitanDBImpl::Open(const std::vector<TitanCFDescriptor>& descs,
 
   // Initialize GC thread pool.
   if (!db_options_.disable_background_gc && db_options_.max_background_gc > 0) {
-    env_->IncBackgroundThreadsIfNeeded(db_options_.max_background_gc,
-                                       Env::Priority::BOTTOM);
+    auto pool = NewThreadPool(0);
+    // Hack: set thread priority to change the thread name
+    (reinterpret_cast<ThreadPoolImpl*>(pool))
+        ->SetThreadPriority(Env::Priority::USER);
+    pool->SetBackgroundThreads(db_options_.max_background_gc);
+    thread_pool_.reset(pool);
   }
 
   s = DB::Open(db_options_, dbname_, base_descs, handles, &db_);
@@ -306,10 +310,13 @@ Status TitanDBImpl::CloseImpl() {
     shuting_down_.store(true, std::memory_order_release);
   }
 
-  int gc_unscheduled = env_->UnSchedule(this, Env::Priority::BOTTOM);
+  if (thread_pool_ != nullptr) {
+    thread_pool_->JoinAllThreads();
+  }
+
   {
     MutexLock l(&mutex_);
-    bg_gc_scheduled_ -= gc_unscheduled;
+    // `bg_gc_scheduled_` should be 0 after `JoinAllThreads`, double check here.
     while (bg_gc_scheduled_ > 0) {
       bg_cv_.Wait();
     }

src/db_impl.h

@@ -2,6 +2,7 @@
 
 #include "db/db_impl.h"
 #include "rocksdb/statistics.h"
+#include "rocksdb/threadpool.h"
 #include "util/repeatable_thread.h"
 
 #include "blob_file_manager.h"
@@ -240,6 +241,9 @@ class TitanDBImpl : public TitanDB {
   Status bg_error_;
   std::atomic_bool has_bg_error_{false};
 
+  // Thread pool for running background GC.
+  std::unique_ptr<ThreadPool> thread_pool_;
+
   // TitanStats is turned on only if statistics field of DBOptions
   // is not null.
   std::unique_ptr<TitanStats> stats_;

src/db_impl_gc.cc

@@ -20,7 +20,7 @@ void TitanDBImpl::MaybeScheduleGC() {
          bg_gc_scheduled_ < db_options_.max_background_gc) {
     unscheduled_gc_--;
     bg_gc_scheduled_++;
-    env_->Schedule(&TitanDBImpl::BGWorkGC, this, Env::Priority::BOTTOM, this);
+    thread_pool_->SubmitJob(std::bind(&TitanDBImpl::BGWorkGC, this));
   }
 }