Cherry-pick titan stats related PRs to tikv-4.x (#490)

* Add titan stats enum (#414)

* add titan customized stats
Signed-off-by: Connor <zbk602423539@gmail.com>

* copy rocksdb options to titan options when using (#488)
Signed-off-by: Connor1996 <zbk602423539@gmail.com>

librocksdb_sys/crocksdb/c.cc

@@ -5459,20 +5459,14 @@ struct ctitandb_options_t {
   TitanOptions rep;
 };
 
-// TODO: Simplify the API by merging db_options into tdb_options, and
-// column_family_options into titan_column_family_options, since the later
-// of the pairs already contain the former.
 crocksdb_t* ctitandb_open_column_families(
-    const char* name, const crocksdb_options_t* db_options,
+    const char* name, 
     const ctitandb_options_t* tdb_options, int num_column_families,
     const char** column_family_names,
-    const crocksdb_options_t** column_family_options,
     const ctitandb_options_t** titan_column_family_options,
     crocksdb_column_family_handle_t** column_family_handles, char** errptr) {
   std::vector<TitanCFDescriptor> column_families;
   for (int i = 0; i < num_column_families; i++) {
-    *((ColumnFamilyOptions*)(&titan_column_family_options[i]->rep)) =
-        column_family_options[i]->rep;
     column_families.push_back(
         TitanCFDescriptor(std::string(column_family_names[i]),
                           TitanCFOptions(titan_column_family_options[i]->rep)));
@@ -5480,12 +5474,10 @@ crocksdb_t* ctitandb_open_column_families(
 
   TitanDB* db;
   std::vector<ColumnFamilyHandle*> handles;
-  *(DBOptions*)&tdb_options->rep = db_options->rep;
   if (SaveError(errptr, TitanDB::Open(tdb_options->rep, std::string(name),
                                       column_families, &handles, &db))) {
     return nullptr;
   }
-
   for (size_t i = 0; i < handles.size(); i++) {
     crocksdb_column_family_handle_t* c_handle =
         new crocksdb_column_family_handle_t;
@@ -5503,15 +5495,11 @@ crocksdb_t* ctitandb_open_column_families(
 // use ctitandb_t for titan specific functions.
 crocksdb_column_family_handle_t* ctitandb_create_column_family(
     crocksdb_t* db,
-    const crocksdb_options_t* column_family_options,
     const ctitandb_options_t* titan_column_family_options,
     const char* column_family_name,
     char** errptr) {
   // Blindly cast db into TitanDB.
   TitanDB* titan_db = reinterpret_cast<TitanDB*>(db->rep);
-  // Copy the ColumnFamilyOptions part of `column_family_options` into `titan_column_family_options`
-  *((ColumnFamilyOptions*)(&titan_column_family_options->rep)) = 
-      column_family_options->rep;
   crocksdb_column_family_handle_t* handle = new crocksdb_column_family_handle_t;
   SaveError(errptr,
       titan_db->CreateColumnFamily(
@@ -5534,6 +5522,11 @@ ctitandb_options_t* ctitandb_options_copy(ctitandb_options_t* src) {
   return new ctitandb_options_t{src->rep};
 }
 
+void ctitandb_options_set_rocksdb_options(ctitandb_options_t* opts, const crocksdb_options_t* rocksdb_opts) {
+  *(DBOptions*)&opts->rep = rocksdb_opts->rep;
+  *(ColumnFamilyOptions*)&opts->rep = rocksdb_opts->rep;
+}
+
 ctitandb_options_t* ctitandb_get_titan_options_cf(
     const crocksdb_t* db,
     crocksdb_column_family_handle_t* column_family) {
@@ -5543,6 +5536,13 @@ ctitandb_options_t* ctitandb_get_titan_options_cf(
   return options;
 }
 
+ctitandb_options_t* ctitandb_get_titan_db_options(crocksdb_t* db) {
+  ctitandb_options_t* options = new ctitandb_options_t;
+  TitanDB* titan_db = reinterpret_cast<TitanDB*>(db->rep);
+  *static_cast<TitanDBOptions*>(&options->rep) = titan_db->GetTitanDBOptions();
+  return options;
+}
+
 const char* ctitandb_options_dirname(ctitandb_options_t* opts) {
   return opts->rep.dirname.c_str();
 }

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -2146,17 +2146,15 @@ typedef struct ctitandb_readoptions_t ctitandb_readoptions_t;
 typedef struct ctitandb_blob_index_t ctitandb_blob_index_t;
 
 extern C_ROCKSDB_LIBRARY_API crocksdb_t* ctitandb_open_column_families(
-    const char* name, const crocksdb_options_t* options,
+    const char* name, 
     const ctitandb_options_t* tdb_options, int num_column_families,
     const char** column_family_names,
-    const crocksdb_options_t** column_family_options,
     const ctitandb_options_t** titan_column_family_options,
     crocksdb_column_family_handle_t** column_family_handles, char** errptr);
 
 extern C_ROCKSDB_LIBRARY_API
 crocksdb_column_family_handle_t* ctitandb_create_column_family(
     crocksdb_t* db,
-    const crocksdb_options_t* column_family_options,
     const ctitandb_options_t* titan_column_family_options,
     const char* column_family_name,
     char** errptr);
@@ -2170,9 +2168,13 @@ extern C_ROCKSDB_LIBRARY_API void ctitandb_options_destroy(ctitandb_options_t*);
 extern C_ROCKSDB_LIBRARY_API ctitandb_options_t* ctitandb_options_copy(
     ctitandb_options_t*);
 
+extern C_ROCKSDB_LIBRARY_API void ctitandb_options_set_rocksdb_options(ctitandb_options_t* opts, const crocksdb_options_t* rocksdb_opts);
+
 extern C_ROCKSDB_LIBRARY_API ctitandb_options_t* ctitandb_get_titan_options_cf(
     const crocksdb_t* db, crocksdb_column_family_handle_t* column_family);
 
+extern C_ROCKSDB_LIBRARY_API ctitandb_options_t* ctitandb_get_titan_db_options(crocksdb_t* db);
+
 extern C_ROCKSDB_LIBRARY_API const char* ctitandb_options_dirname(
     ctitandb_options_t*);
 

titan @ cd8eb48c

-Subproject commit 2874b2c4e48760fc41c5ac728d6f57dbe1c17532
+Subproject commit cd8eb48c29834602e4df3141eff7b23f5a1345ed

librocksdb_sys/src/generated.rs

@@ -149,6 +149,33 @@ pub enum DBStatisticsTickerType {
     BlockCacheCompressionDictAdd = 142,
     BlockCacheCompressionDictBytesInsert = 143,
     BlockCacheCompressionDictBytesEvict = 144,
+    TitanNumGet = 145,
+    TitanNumSeek,
+    TitanNumNext,
+    TitanNumPrev,
+    TitanBlobFileNumKeysWritten,
+    TitanBlobFileNumKeysRead,
+    TitanBlobFileBytesWritten,
+    TitanBlobFileBytesRead,
+    TitanBlobFileSynced,
+    TitanGcNumFiles,
+    TitanGcNumNewFiles,
+    TitanGcNumKeysOverwritten,
+    TitanGcNumKeysRelocated,
+    TitanGcBytesOverwritten,
+    TitanGcBytesRelocated,
+    TitanGcBytesWritten,
+    TitanGcBytesRead,
+    TitanBlobCacheHit,
+    TitanBlobCacheMiss,
+    TitanGcNoNeed,
+    TitanGcRemain,
+    TitanGcDiscardable,
+    TitanGcSample,
+    TitanGcSmallFile,
+    TitanGcFailure,
+    TitanGcSuccess,
+    TitanGcTriggerNext,
 }
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[repr(C)]
@@ -202,5 +229,18 @@ pub enum DBStatisticsHistogramType {
     FlushTime = 46,
     SstBatchSize = 47,
     DbWriteWalTime = 48,
-    HistogramEnumMax = 49,
+    TitanKeySize = 49,
+    TitanValueSize,
+    TitanGetMicros,
+    TitanSeekMicros,
+    TitanNextMicros,
+    TitanPrevMicros,
+    TitanBlobFileWriteMicros,
+    TitanBlobFileReadMicros,
+    TitanBlobFileSyncMicros,
+    TitanManifestFileSyncMicros,
+    TitanGcMicros,
+    TitanGcInputFileSize,
+    TitanGcOutputFileSize,
+    TitanIterTouchBlobFileCount,
 }

librocksdb_sys/src/lib.rs

@@ -2163,11 +2163,9 @@ extern "C" {
 extern "C" {
     pub fn ctitandb_open_column_families(
         path: *const c_char,
-        options: *const Options,
         titan_options: *const DBTitanDBOptions,
         num_column_families: c_int,
         column_family_names: *const *const c_char,
-        column_family_options: *const *const Options,
         titan_column_family_options: *const *const DBTitanDBOptions,
         column_family_handles: *const *mut DBCFHandle,
         err: *mut *mut c_char,
@@ -2175,7 +2173,6 @@ extern "C" {
 
     pub fn ctitandb_create_column_family(
         db: *mut DBInstance,
-        column_family_options: *const Options,
         titan_column_family_options: *const DBTitanDBOptions,
         column_family_name: *const c_char,
         err: *mut *mut c_char,
@@ -2184,10 +2181,15 @@ extern "C" {
     pub fn ctitandb_options_create() -> *mut DBTitanDBOptions;
     pub fn ctitandb_options_destroy(opts: *mut DBTitanDBOptions);
     pub fn ctitandb_options_copy(opts: *mut DBTitanDBOptions) -> *mut DBTitanDBOptions;
+    pub fn ctitandb_options_set_rocksdb_options(
+        opts: *mut DBTitanDBOptions,
+        rocksdb_opts: *const Options,
+    );
     pub fn ctitandb_get_titan_options_cf(
         db: *mut DBInstance,
         cf: *mut DBCFHandle,
     ) -> *mut DBTitanDBOptions;
+    pub fn ctitandb_get_titan_db_options(db: *mut DBInstance) -> *mut DBTitanDBOptions;
     pub fn ctitandb_options_dirname(opts: *mut DBTitanDBOptions) -> *const c_char;
     pub fn ctitandb_options_set_dirname(opts: *mut DBTitanDBOptions, name: *const c_char);
     pub fn ctitandb_options_min_blob_size(opts: *mut DBTitanDBOptions) -> u64;

src/rocksdb.rs

@@ -83,11 +83,15 @@ fn ensure_default_cf_exists<'a>(
 
 fn split_descriptors<'a>(
     list: Vec<ColumnFamilyDescriptor<'a>>,
+    is_titan: bool,
 ) -> (Vec<&'a str>, Vec<ColumnFamilyOptions>) {
     let mut v1 = Vec::with_capacity(list.len());
     let mut v2 = Vec::with_capacity(list.len());
-    for d in list {
+    for mut d in list {
         v1.push(d.name);
+        if is_titan && d.options.titan_inner.is_null() {
+            d.options.set_titandb_options(&TitanDBOptions::new());
+        }
         v2.push(d.options);
     }
     (v1, v2)
@@ -541,7 +545,7 @@ impl DB {
     }
 
     fn open_cf_internal<'a, T>(
-        opts: DBOptions,
+        mut opts: DBOptions,
         path: &str,
         cfds: Vec<T>,
         ttls: &[i32],
@@ -570,7 +574,7 @@ impl DB {
         let mut ttls_vec = ttls.to_vec();
         ensure_default_cf_exists(&mut descs, &mut ttls_vec, !opts.titan_inner.is_null());
 
-        let (names, options) = split_descriptors(descs);
+        let (names, options) = split_descriptors(descs, !opts.titan_inner.is_null());
         let cstrings = build_cstring_list(&names);
 
         let cf_names: Vec<*const _> = cstrings.iter().map(|cs| cs.as_ptr()).collect();
@@ -581,7 +585,14 @@ impl DB {
             .collect();
         let titan_cf_options: Vec<_> = options
             .iter()
-            .map(|x| x.titan_inner as *const crocksdb_ffi::DBTitanDBOptions)
+            .map(|x| {
+                if !x.titan_inner.is_null() {
+                    unsafe {
+                        crocksdb_ffi::ctitandb_options_set_rocksdb_options(x.titan_inner, x.inner);
+                    }
+                }
+                x.titan_inner as *const crocksdb_ffi::DBTitanDBOptions
+            })
             .collect();
 
         let readonly = error_if_log_file_exist.is_some();
@@ -607,6 +618,9 @@ impl DB {
 
             let titan_options = opts.titan_inner;
             if !titan_options.is_null() {
+                unsafe {
+                    crocksdb_ffi::ctitandb_options_set_rocksdb_options(titan_options, db_options);
+                }
                 if error_if_log_file_exist.is_some() {
                     return Err("TitanDB doesn't support read only mode.".to_owned());
                 } else if with_ttl {
@@ -642,11 +656,9 @@ impl DB {
                     unsafe {
                         ffi_try!(ctitandb_open_column_families(
                             db_path,
-                            db_options,
                             titan_options,
                             db_cfs_count,
                             db_cf_ptrs,
-                            db_cf_opts,
                             titan_cf_opts,
                             db_cf_handles
                         ))
@@ -677,6 +689,16 @@ impl DB {
             return Err(ERR_NULL_DB_ONINIT.to_owned());
         }
 
+        unsafe {
+            // the options provided when opening db may sanitized, so get the latest options.
+            crocksdb_ffi::crocksdb_options_destroy(opts.inner);
+            opts.inner = crocksdb_ffi::crocksdb_get_db_options(db);
+            if !opts.titan_inner.is_null() {
+                crocksdb_ffi::ctitandb_options_destroy(opts.titan_inner);
+                opts.titan_inner = crocksdb_ffi::ctitandb_get_titan_db_options(db);
+            }
+        }
+
         let cfs = names
             .into_iter()
             .zip(cf_handles)
@@ -847,7 +869,7 @@ impl DB {
     where
         T: Into<ColumnFamilyDescriptor<'a>>,
     {
-        let cfd = cfd.into();
+        let mut cfd = cfd.into();
         let cname = match CString::new(cfd.name.as_bytes()) {
             Ok(c) => c,
             Err(_) => {
@@ -863,9 +885,15 @@ impl DB {
                     cname_ptr
                 ))
             } else {
+                if cfd.options.titan_inner.is_null() {
+                    cfd.options.set_titandb_options(&TitanDBOptions::new());
+                }
+                crocksdb_ffi::ctitandb_options_set_rocksdb_options(
+                    cfd.options.titan_inner,
+                    cfd.options.inner,
+                );
                 ffi_try!(ctitandb_create_column_family(
                     self.inner,
-                    cfd.options.inner,
                     cfd.options.titan_inner,
                     cname_ptr
                 ))

src/rocksdb_options.rs

@@ -695,7 +695,9 @@ impl DBOptions {
     }
 
     pub fn set_titandb_options(&mut self, opts: &TitanDBOptions) {
-        self.titan_inner = unsafe { crocksdb_ffi::ctitandb_options_copy(opts.inner) }
+        unsafe {
+            self.titan_inner = crocksdb_ffi::ctitandb_options_copy(opts.inner);
+        }
     }
 
     pub fn increase_parallelism(&mut self, parallelism: i32) {
@@ -1206,8 +1208,8 @@ impl ColumnFamilyOptions {
     }
 
     pub fn set_titandb_options(&mut self, opts: &TitanDBOptions) {
-        if !opts.inner.is_null() {
-            self.titan_inner = unsafe { crocksdb_ffi::ctitandb_options_copy(opts.inner) }
+        unsafe {
+            self.titan_inner = crocksdb_ffi::ctitandb_options_copy(opts.inner);
         }
     }
 

tests/cases/test_titan.rs

@@ -17,9 +17,10 @@ use std::ops;
 use rand::Rng;
 use rocksdb::{
     CFHandle, ColumnFamilyOptions, CompactOptions, DBBottommostLevelCompaction, DBCompressionType,
-    DBEntryType, DBOptions, Range, ReadOptions, SeekKey, TablePropertiesCollector,
+    DBEntryType, DBOptions, DBStatisticsHistogramType as HistogramType,
+    DBStatisticsTickerType as TickerType, Range, ReadOptions, SeekKey, TablePropertiesCollector,
     TablePropertiesCollectorFactory, TitanBlobIndex, TitanDBOptions, UserCollectedProperties,
-    Writable, DB,
+    Writable, WriteOptions, DB,
 };
 
 use super::tempdir_with_prefix;
@@ -119,8 +120,8 @@ fn test_titandb() {
     opts.set_titandb_options(&tdb_opts);
     let mut cf_opts = ColumnFamilyOptions::new();
     let f = TitanCollectorFactory::default();
-    cf_opts.add_table_properties_collector_factory("titan-collector", Box::new(f));
     cf_opts.set_titandb_options(&tdb_opts);
+    cf_opts.add_table_properties_collector_factory("titan-collector", Box::new(f));
 
     let mut db = DB::open_cf(
         opts,
@@ -141,7 +142,6 @@ fn test_titandb() {
 
     let mut cf_opts = ColumnFamilyOptions::new();
     cf_opts.set_num_levels(4);
-    cf_opts.set_titandb_options(&tdb_opts);
     db.create_cf(("cf1", cf_opts)).unwrap();
     let cf1 = db.cf_handle("cf1").unwrap();
     assert_eq!(db.get_options_cf(cf1).get_num_levels(), 4);
@@ -373,3 +373,64 @@ fn test_blob_cache_capacity() {
 
     assert_eq!(db.get_options().get_blob_cache_capacity(), 32 * 1024 * 1024);
 }
+
+#[test]
+fn test_titan_statistics() {
+    let path = tempdir_with_prefix("_rust_rocksdb_statistics");
+    let mut tdb_opts = TitanDBOptions::new();
+    tdb_opts.set_min_blob_size(0);
+    let mut opts = DBOptions::new();
+    opts.set_titandb_options(&tdb_opts);
+    opts.enable_statistics(true);
+    opts.create_if_missing(true);
+    let mut cf_opts = ColumnFamilyOptions::new();
+    cf_opts.set_titandb_options(&tdb_opts);
+
+    let db = DB::open_cf(
+        opts,
+        path.path().to_str().unwrap(),
+        vec![("default", cf_opts)],
+    )
+    .unwrap();
+    let wopts = WriteOptions::new();
+
+    db.put_opt(b"k0", b"a", &wopts).unwrap();
+    db.put_opt(b"k1", b"b", &wopts).unwrap();
+    db.put_opt(b"k2", b"c", &wopts).unwrap();
+    db.flush(true /* sync */).unwrap(); // flush memtable to sst file.
+    assert_eq!(db.get(b"k0").unwrap().unwrap(), b"a");
+    assert_eq!(db.get(b"k1").unwrap().unwrap(), b"b");
+    assert_eq!(db.get(b"k2").unwrap().unwrap(), b"c");
+
+    assert!(db.get_statistics_ticker_count(TickerType::TitanNumGet) > 0);
+    assert!(db.get_and_reset_statistics_ticker_count(TickerType::TitanNumGet) > 0);
+    assert_eq!(db.get_statistics_ticker_count(TickerType::TitanNumGet), 0);
+    assert!(db
+        .get_statistics_histogram_string(HistogramType::TitanGetMicros)
+        .is_some());
+    assert!(db
+        .get_statistics_histogram(HistogramType::TitanGetMicros)
+        .is_some());
+
+    let get_micros = db
+        .get_statistics_histogram(HistogramType::TitanGetMicros)
+        .unwrap();
+    assert!(get_micros.max > 0.0);
+    db.reset_statistics();
+    let get_micros = db
+        .get_statistics_histogram(HistogramType::TitanGetMicros)
+        .unwrap();
+    assert_eq!(
+        db.get_property_int("rocksdb.titandb.num-discardable-ratio-le0-file")
+            .unwrap(),
+        1
+    );
+
+    assert_eq!(
+        db.get_property_int("rocksdb.titandb.num-blob-files-at-level0")
+            .unwrap(),
+        1
+    );
+
+    assert_eq!(get_micros.max, 0.0);
+}