add interface corresponds to the Logger interface (#455)

Signed-off-by: Wangweizhen <hawking.rei@gmail.com>

UCP [#6496](https://github.com/tikv/tikv/issues/6496)

add interface corresponds to the Logger interface 

librocksdb_sys/crocksdb/c.cc

@@ -9,6 +9,11 @@
 
 #include "crocksdb/c.h"
 
+#include <stdlib.h>
+
+#include <limits>
+
+#include "db/column_family.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/comparator.h"
@@ -18,6 +23,7 @@
 #include "rocksdb/env.h"
 #include "rocksdb/env_encryption.h"
 #include "rocksdb/filter_policy.h"
+#include "rocksdb/iostats_context.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/ldb_tool.h"
 #include "rocksdb/listener.h"
@@ -38,22 +44,14 @@
 #include "rocksdb/utilities/debug.h"
 #include "rocksdb/utilities/options_util.h"
 #include "rocksdb/write_batch.h"
-#include "rocksdb/iostats_context.h"
-
-#include "db/column_family.h"
-#include "table/sst_file_writer_collectors.h"
+#include "src/blob_format.h"
 #include "table/block_based/block_based_table_factory.h"
+#include "table/sst_file_writer_collectors.h"
 #include "table/table_reader.h"
-#include "util/file_reader_writer.h"
-#include "util/coding.h"
-
 #include "titan/db.h"
 #include "titan/options.h"
-#include "src/blob_format.h"
-
-#include <stdlib.h>
-
-#include <limits>
+#include "util/coding.h"
+#include "util/file_reader_writer.h"
 
 #if !defined(ROCKSDB_MAJOR) || !defined(ROCKSDB_MINOR) || !defined(ROCKSDB_PATCH)
 #error Only rocksdb 5.7.3+ is supported.
@@ -222,6 +220,24 @@ struct crocksdb_randomfile_t      { RandomAccessFile* rep; };
 struct crocksdb_writablefile_t    { WritableFile*     rep; };
 struct crocksdb_filelock_t        { FileLock*         rep; };
 struct crocksdb_logger_t          { shared_ptr<Logger>  rep; };
+struct crocksdb_logger_impl_t : public Logger {
+  void* rep;
+
+  void (*destructor_)(void*);
+  void (*logv_internal_)(void* logger, int log_level, const char* format,
+                         va_list ap);
+
+  void Logv(const char* format, va_list ap) override {
+    logv_internal_(rep, InfoLogLevel::INFO_LEVEL, format, ap);
+  }
+
+  void Logv(const InfoLogLevel log_level, const char* format,
+            va_list ap) override {
+    logv_internal_(rep, log_level, format, ap);
+  }
+
+  virtual ~crocksdb_logger_impl_t() { (*destructor_)(rep); }
+};
 struct crocksdb_lru_cache_options_t {
   LRUCacheOptions rep;
 };
@@ -2323,7 +2339,19 @@ void crocksdb_options_set_env(crocksdb_options_t* opt, crocksdb_env_t* env) {
   opt->rep.env = (env ? env->rep : nullptr);
 }
 
-void crocksdb_options_set_info_log(crocksdb_options_t* opt, crocksdb_logger_t* l) {
+crocksdb_logger_t* crocksdb_logger_create(void* rep, void (*destructor_)(void*),
+                                          crocksdb_logger_logv_cb logv) {
+  crocksdb_logger_t* logger = new crocksdb_logger_t;
+  crocksdb_logger_impl_t* li = new crocksdb_logger_impl_t;
+  li->rep = rep;
+  li->destructor_ = destructor_;
+  li->logv_internal_ = logv;
+  logger->rep = std::shared_ptr<Logger>(li);
+  return logger;
+}
+
+void crocksdb_options_set_info_log(crocksdb_options_t* opt,
+                                   crocksdb_logger_t* l) {
   if (l) {
     opt->rep.info_log = l->rep;
   }
@@ -2638,7 +2666,8 @@ void crocksdb_options_set_enable_multi_batch_write(crocksdb_options_t *opt,
   opt->rep.enable_multi_thread_write = v;
 }
 
-unsigned char crocksdb_options_is_enable_multi_batch_write(crocksdb_options_t *opt) {
+unsigned char crocksdb_options_is_enable_multi_batch_write(
+    crocksdb_options_t* opt) {
   return opt->rep.enable_multi_thread_write;
 }
 
@@ -3649,7 +3678,8 @@ void crocksdb_sequential_file_destroy(crocksdb_sequential_file_t* file) {
 
 #ifdef OPENSSL
 crocksdb_file_encryption_info_t* crocksdb_file_encryption_info_create() {
-  crocksdb_file_encryption_info_t* file_info = new crocksdb_file_encryption_info_t;
+  crocksdb_file_encryption_info_t* file_info =
+      new crocksdb_file_encryption_info_t;
   file_info->rep = new FileEncryptionInfo;
   return file_info;
 }
@@ -3699,7 +3729,8 @@ const char* crocksdb_file_encryption_info_iv(
 }
 
 void crocksdb_file_encryption_info_set_method(
-    crocksdb_file_encryption_info_t* file_info, crocksdb_encryption_method_t method) {
+    crocksdb_file_encryption_info_t* file_info,
+    crocksdb_encryption_method_t method) {
   assert(file_info != nullptr);
   switch (method) {
     case kUnknown:
@@ -3723,7 +3754,8 @@ void crocksdb_file_encryption_info_set_method(
 }
 
 void crocksdb_file_encryption_info_set_key(
-    crocksdb_file_encryption_info_t* file_info, const char* key, size_t keylen) {
+    crocksdb_file_encryption_info_t* file_info, const char* key,
+    size_t keylen) {
   assert(file_info != nullptr);
   file_info->rep->key = std::string(key, keylen);
 }
@@ -3743,12 +3775,10 @@ struct crocksdb_encryption_key_manager_impl_t : public KeyManager {
   crocksdb_encryption_key_manager_link_file_cb link_file;
   crocksdb_encryption_key_manager_rename_file_cb rename_file;
 
-  virtual ~crocksdb_encryption_key_manager_impl_t() {
-    destructor(state);
-  }
+  virtual ~crocksdb_encryption_key_manager_impl_t() { destructor(state); }
 
-  Status GetFile(
-      const std::string& fname, FileEncryptionInfo* file_info) override {
+  Status GetFile(const std::string& fname,
+                 FileEncryptionInfo* file_info) override {
     crocksdb_file_encryption_info_t info;
     info.rep = file_info;
     const char* ret = get_file(state, fname.c_str(), &info);
@@ -3759,9 +3789,9 @@ struct crocksdb_encryption_key_manager_impl_t : public KeyManager {
     }
     return s;
   }
-  
-  Status NewFile(
-      const std::string& fname, FileEncryptionInfo* file_info) override {
+
+  Status NewFile(const std::string& fname,
+                 FileEncryptionInfo* file_info) override {
     crocksdb_file_encryption_info_t info;
     info.rep = file_info;
     const char* ret = new_file(state, fname.c_str(), &info);
@@ -3772,7 +3802,7 @@ struct crocksdb_encryption_key_manager_impl_t : public KeyManager {
     }
     return s;
   }
-  
+
   Status DeleteFile(const std::string& fname) override {
     const char* ret = delete_file(state, fname.c_str());
     Status s;
@@ -3783,8 +3813,8 @@ struct crocksdb_encryption_key_manager_impl_t : public KeyManager {
     return s;
   }
 
-  Status LinkFile(
-      const std::string& src_fname, const std::string& dst_fname) override {
+  Status LinkFile(const std::string& src_fname,
+                  const std::string& dst_fname) override {
     const char* ret = link_file(state, src_fname.c_str(), dst_fname.c_str());
     Status s;
     if (ret != nullptr) {
@@ -3794,8 +3824,8 @@ struct crocksdb_encryption_key_manager_impl_t : public KeyManager {
     return s;
   }
 
-  Status RenameFile(
-      const std::string& src_fname, const std::string& dst_fname) override {
+  Status RenameFile(const std::string& src_fname,
+                    const std::string& dst_fname) override {
     const char* ret = rename_file(state, src_fname.c_str(), dst_fname.c_str());
     Status s;
     if (ret != nullptr) {
@@ -3822,16 +3852,17 @@ crocksdb_encryption_key_manager_t* crocksdb_encryption_key_manager_create(
   key_manager_impl->delete_file = delete_file;
   key_manager_impl->link_file = link_file;
   key_manager_impl->rename_file = rename_file;
-  crocksdb_encryption_key_manager_t* key_manager = new crocksdb_encryption_key_manager_t;
+  crocksdb_encryption_key_manager_t* key_manager =
+      new crocksdb_encryption_key_manager_t;
   key_manager->rep = key_manager_impl;
   return key_manager;
 }
 
-void crocksdb_encryption_key_manager_destroy(crocksdb_encryption_key_manager_t* key_manager) {
+void crocksdb_encryption_key_manager_destroy(
+    crocksdb_encryption_key_manager_t* key_manager) {
   delete key_manager;
 }
 
-
 const char* crocksdb_encryption_key_manager_get_file(
     crocksdb_encryption_key_manager_t* key_manager, const char* fname,
     crocksdb_file_encryption_info_t* file_info) {
@@ -4400,6 +4431,17 @@ void crocksdb_delete_files_in_ranges_cf(
 
 void crocksdb_free(void* ptr) { free(ptr); }
 
+crocksdb_logger_t* crocksdb_create_env_logger(const char* fname,
+                                              crocksdb_env_t* env) {
+  crocksdb_logger_t* logger = new crocksdb_logger_t;
+  Status s = NewEnvLogger(std::string(fname), env->rep, &logger->rep);
+  if (!s.ok()) {
+    delete logger;
+    return NULL;
+  }
+  return logger;
+}
+
 crocksdb_logger_t *crocksdb_create_log_from_options(const char *path,
                                                     crocksdb_options_t *opts,
                                                     char **errptr) {
@@ -5562,7 +5604,7 @@ void ctitandb_options_set_blob_file_compression(ctitandb_options_t* opts,
 }
 
 void ctitandb_options_set_gc_merge_rewrite(ctitandb_options_t* opts,
-                                               unsigned char enable) {
+                                           unsigned char enable) {
   opts->rep.gc_merge_rewrite = enable;
 }
 

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -91,6 +91,7 @@ typedef struct crocksdb_filterpolicy_t    crocksdb_filterpolicy_t;
 typedef struct crocksdb_flushoptions_t    crocksdb_flushoptions_t;
 typedef struct crocksdb_iterator_t        crocksdb_iterator_t;
 typedef struct crocksdb_logger_t          crocksdb_logger_t;
+typedef struct crocksdb_logger_impl_t crocksdb_logger_impl_t;
 typedef struct crocksdb_mergeoperator_t   crocksdb_mergeoperator_t;
 typedef struct crocksdb_options_t         crocksdb_options_t;
 typedef struct crocksdb_column_family_descriptor
@@ -191,7 +192,8 @@ typedef enum crocksdb_encryption_method_t {
 } crocksdb_encryption_method_t;
 
 typedef struct crocksdb_file_encryption_info_t crocksdb_file_encryption_info_t;
-typedef struct crocksdb_encryption_key_manager_t crocksdb_encryption_key_manager_t;
+typedef struct crocksdb_encryption_key_manager_t
+    crocksdb_encryption_key_manager_t;
 #endif
 
 /* DB operations */
@@ -807,6 +809,8 @@ typedef void (*on_external_file_ingested_cb)(
 typedef void (*on_background_error_cb)(void*, crocksdb_backgrounderrorreason_t,
                                        crocksdb_status_ptr_t*);
 typedef void (*on_stall_conditions_changed_cb)(void*, const crocksdb_writestallinfo_t*);
+typedef void (*crocksdb_logger_logv_cb)(void*, int log_level, const char*,
+                                        va_list);
 
 extern C_ROCKSDB_LIBRARY_API crocksdb_eventlistener_t*
 crocksdb_eventlistener_create(
@@ -892,6 +896,8 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_paranoid_checks(
     crocksdb_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_env(crocksdb_options_t*,
                                                         crocksdb_env_t*);
+extern C_ROCKSDB_LIBRARY_API crocksdb_logger_t* crocksdb_logger_create(
+    void* rep, void (*destructor_)(void*), crocksdb_logger_logv_cb logv);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_info_log(crocksdb_options_t*,
                                                              crocksdb_logger_t*);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_info_log_level(
@@ -1071,7 +1077,7 @@ extern C_ROCKSDB_LIBRARY_API void
 crocksdb_options_set_enable_multi_batch_write(crocksdb_options_t *opt,
                                              unsigned char v);
 extern C_ROCKSDB_LIBRARY_API unsigned char
-crocksdb_options_is_enable_multi_batch_write(crocksdb_options_t *opt);
+crocksdb_options_is_enable_multi_batch_write(crocksdb_options_t* opt);
 extern C_ROCKSDB_LIBRARY_API void
 crocksdb_options_set_unordered_write(crocksdb_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void
@@ -1460,13 +1466,15 @@ crocksdb_file_encryption_info_create();
 extern C_ROCKSDB_LIBRARY_API void crocksdb_file_encryption_info_destroy(
     crocksdb_file_encryption_info_t* file_info);
 extern C_ROCKSDB_LIBRARY_API crocksdb_encryption_method_t
-crocksdb_file_encryption_info_method(crocksdb_file_encryption_info_t* file_info);
+crocksdb_file_encryption_info_method(
+    crocksdb_file_encryption_info_t* file_info);
 extern C_ROCKSDB_LIBRARY_API const char* crocksdb_file_encryption_info_key(
     crocksdb_file_encryption_info_t* file_info, size_t* keylen);
 extern C_ROCKSDB_LIBRARY_API const char* crocksdb_file_encryption_info_iv(
     crocksdb_file_encryption_info_t* file_info, size_t* ivlen);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_file_encryption_info_set_method(
-    crocksdb_file_encryption_info_t* file_info, crocksdb_encryption_method_t method);
+    crocksdb_file_encryption_info_t* file_info,
+    crocksdb_encryption_method_t method);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_file_encryption_info_set_key(
     crocksdb_file_encryption_info_t* file_info, const char* key, size_t keylen);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_file_encryption_info_set_iv(
@@ -1514,8 +1522,8 @@ crocksdb_encryption_key_manager_rename_file(
     const char* dst_fname);
 
 extern C_ROCKSDB_LIBRARY_API crocksdb_env_t*
-crocksdb_key_managed_encrypted_env_create(
-    crocksdb_env_t*, crocksdb_encryption_key_manager_t*);
+crocksdb_key_managed_encrypted_env_create(crocksdb_env_t*,
+                                          crocksdb_encryption_key_manager_t*);
 #endif
 
 /* SstFile */
@@ -1725,6 +1733,8 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_delete_files_in_ranges_cf(
 // to free memory that was malloc()ed
 extern C_ROCKSDB_LIBRARY_API void crocksdb_free(void* ptr);
 
+extern C_ROCKSDB_LIBRARY_API crocksdb_logger_t* crocksdb_create_env_logger(
+    const char* fname, crocksdb_env_t* env);
 extern C_ROCKSDB_LIBRARY_API crocksdb_logger_t *
 crocksdb_create_log_from_options(const char *path, crocksdb_options_t *opts,
                                  char **errptr);

librocksdb_sys/src/lib.rs

@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#![feature(c_variadic)]
 
 extern crate bzip2_sys;
 extern crate libc;
@@ -18,6 +19,7 @@ extern crate libc;
 extern crate tempfile;
 
 use std::ffi::CStr;
+use std::ffi::VaList;
 use std::fmt;
 
 use libc::{c_char, c_double, c_int, c_uchar, c_void, size_t};
@@ -1710,13 +1712,24 @@ extern "C" {
         name: extern "C" fn(*mut c_void) -> *const c_char,
     ) -> *mut DBSliceTransform;
     pub fn crocksdb_slicetransform_destroy(transform: *mut DBSliceTransform);
+    pub fn crocksdb_logger_create(
+        state: *mut c_void,
+        destructor: extern "C" fn(*mut c_void),
+        logv: extern "C" fn(
+            ctx: *mut c_void,
+            log_level: DBInfoLogLevel,
+            format: *const c_char,
+            ap: VaList,
+        ),
+    ) -> *mut DBLogger;
+    pub fn crocksdb_create_env_logger(fname: *const libc::c_char, env: *mut DBEnv)
+        -> *mut DBLogger;
     pub fn crocksdb_create_log_from_options(
         path: *const c_char,
         options: *mut Options,
         err: *mut *mut c_char,
     ) -> *mut DBLogger;
     pub fn crocksdb_log_destroy(logger: *mut DBLogger);
-
     pub fn crocksdb_get_pinned(
         db: *mut DBInstance,
         readopts: *const DBReadOptions,

src/lib.rs

@@ -16,6 +16,7 @@
 // FIXME: we should remove this line after we add safe doc to all the unsafe functions
 // see: https://rust-lang.github.io/rust-clippy/master/index.html#missing_safety_doc
 #![allow(clippy::missing_safety_doc)]
+#![feature(c_variadic)]
 
 extern crate core;
 extern crate libc;
@@ -40,6 +41,7 @@ pub use librocksdb_sys::{
     DBEntryType, DBInfoLogLevel, DBRateLimiterMode, DBRecoveryMode, DBStatisticsHistogramType,
     DBStatisticsTickerType, DBStatusPtr, DBTitanDBBlobRunMode, IndexType, WriteStallCondition,
 };
+pub use logger::Logger;
 pub use merge_operator::MergeOperands;
 pub use metadata::{ColumnFamilyMetaData, LevelMetaData, SstFileMetaData};
 pub use perf_context::{get_perf_level, set_perf_level, IOStatsContext, PerfContext, PerfLevel};
@@ -72,6 +74,7 @@ pub mod comparator;
 #[cfg(feature = "encryption")]
 mod encryption;
 mod event_listener;
+pub mod logger;
 pub mod merge_operator;
 mod metadata;
 mod perf_context;

src/logger.rs

+// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+
+use crocksdb_ffi;
+use libc::{c_char, c_void};
+use librocksdb_sys::{DBEnv, DBInfoLogLevel as InfoLogLevel, DBLogger};
+use std::ffi::{CString, VaList};
+
+pub trait Logger: Send + Sync {
+    fn logv(&self, log_level: InfoLogLevel, format: *const c_char, ap: VaList);
+}
+
+extern "C" fn destructor(ctx: *mut c_void) {
+    unsafe {
+        Box::from_raw(ctx as *mut Box<dyn Logger>);
+    }
+}
+
+extern "C" fn logv(ctx: *mut c_void, log_level: InfoLogLevel, format: *const c_char, ap: VaList) {
+    unsafe {
+        let logger = &*(ctx as *mut Box<dyn Logger>);
+        logger.logv(log_level, format, ap);
+    }
+}
+
+pub fn new_logger<L: Logger>(l: L) -> *mut DBLogger {
+    unsafe {
+        let p: Box<dyn Logger> = Box::new(l);
+        crocksdb_ffi::crocksdb_logger_create(
+            Box::into_raw(Box::new(p)) as *mut c_void,
+            destructor,
+            logv,
+        )
+    }
+}
+
+pub fn create_env_logger(fname: &str, mut env: DBEnv) -> *mut DBLogger {
+    let name = CString::new(fname.as_bytes()).unwrap();
+    unsafe { crocksdb_ffi::crocksdb_create_env_logger(name.as_ptr(), &mut env) }
+}

src/rocksdb_options.rs

@@ -28,6 +28,7 @@ use crocksdb_ffi::{
 };
 use event_listener::{new_event_listener, EventListener};
 use libc::{self, c_double, c_int, c_uchar, c_void, size_t};
+use logger::{new_logger, Logger};
 use merge_operator::MergeFn;
 use merge_operator::{self, full_merge_callback, partial_merge_callback, MergeOperatorCallback};
 use rocksdb::Env;
@@ -1025,6 +1026,14 @@ impl DBOptions {
         Ok(())
     }
 
+    // Set the logger to options.
+    pub fn set_info_log<L: Logger>(&self, l: L) {
+        let logger = new_logger(l);
+        unsafe {
+            crocksdb_ffi::crocksdb_options_set_info_log(self.inner, logger);
+        }
+    }
+
     pub fn enable_pipelined_write(&self, v: bool) {
         unsafe {
             crocksdb_ffi::crocksdb_options_set_enable_pipelined_write(self.inner, v);

tests/cases/mod.rs

@@ -8,6 +8,7 @@ mod test_encryption;
 mod test_event_listener;
 mod test_ingest_external_file;
 mod test_iterator;
+mod test_logger;
 mod test_metadata;
 mod test_multithreaded;
 mod test_prefix_extractor;

tests/cases/test_logger.rs

+use std::ffi::VaList;
+use std::sync::atomic::*;
+use std::sync::Arc;
+use std::thread;
+use std::time::Duration;
+
+use super::tempdir_with_prefix;
+use libc::c_char;
+use rocksdb::{DBInfoLogLevel as InfoLogLevel, DBOptions, Logger, DB};
+
+#[derive(Default, Clone)]
+struct TestDrop {
+    called: Arc<AtomicUsize>,
+}
+
+impl Drop for TestDrop {
+    fn drop(&mut self) {
+        self.called.fetch_add(1, Ordering::SeqCst);
+    }
+}
+
+#[derive(Default, Clone)]
+struct TestLogger {
+    print: Arc<AtomicUsize>,
+    drop: Option<TestDrop>,
+}
+
+impl Logger for TestLogger {
+    fn logv(&self, _log_level: InfoLogLevel, _format: *const c_char, _ap: VaList) {
+        self.print.fetch_add(1, Ordering::SeqCst);
+    }
+}
+
+#[test]
+fn test_logger() {
+    let drop_called = Arc::new(AtomicUsize::new(0));
+    let path = tempdir_with_prefix("_rust_rocksdb_test_create_info_rust_log_opt");
+    let mut opts = DBOptions::new();
+    let logger = TestLogger {
+        drop: Some(TestDrop {
+            called: drop_called.clone(),
+        }),
+        print: Default::default(),
+    };
+    opts.set_info_log(logger.clone());
+    opts.create_if_missing(true);
+    opts.set_info_log_level(InfoLogLevel::Debug);
+    let db = DB::open(opts.clone(), path.path().to_str().unwrap()).unwrap();
+    thread::sleep(Duration::from_secs(2));
+    assert_ne!(logger.print.load(Ordering::SeqCst), 0);
+    drop(db);
+    drop(opts);
+    assert_eq!(0, drop_called.load(Ordering::SeqCst));
+    drop(logger);
+    assert_eq!(1, drop_called.load(Ordering::SeqCst));
+}

tests/tests.rs

+#![feature(c_variadic)]
 extern crate crc;
+extern crate libc;
 extern crate rand;
 extern crate rocksdb;
 extern crate tempfile;