export RocksDB statistics (#6)

librocksdb_sys/crocksdb/c.cc

@@ -2005,6 +2005,33 @@ char *crocksdb_options_statistics_get_string(crocksdb_options_t *opt) {
   return nullptr;
 }
 
+uint64_t crocksdb_options_statistics_get_ticker_count(crocksdb_options_t* opt,
+                                                      uint32_t ticker_type) {
+  rocksdb::Statistics* statistics = opt->rep.statistics.get();
+  if (statistics) {
+    return statistics->getTickerCount(ticker_type);
+  }
+  return 0;
+}
+
+uint64_t crocksdb_options_statistics_get_and_reset_ticker_count(crocksdb_options_t* opt,
+                                                                uint32_t ticker_type) {
+  rocksdb::Statistics* statistics = opt->rep.statistics.get();
+  if (statistics) {
+    return statistics->getAndResetTickerCount(ticker_type);
+  }
+  return 0;
+}
+
+char* crocksdb_options_statistics_get_histogram_string(crocksdb_options_t* opt,
+                                                       uint32_t type) {
+  rocksdb::Statistics* statistics = opt->rep.statistics.get();
+  if (statistics) {
+    return strdup(statistics->getHistogramString(type).c_str());
+  }
+  return nullptr;
+}
+
 void crocksdb_options_set_ratelimiter(crocksdb_options_t *opt, crocksdb_ratelimiter_t *limiter) {
   opt->rep.rate_limiter.reset(limiter->rep);
   limiter->rep = nullptr;

librocksdb_sys/crocksdb/rocksdb/c.h

@@ -617,6 +617,13 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_options_enable_statistics(
 /* returns a pointer to a malloc()-ed, null terminated string */
 extern C_ROCKSDB_LIBRARY_API char* crocksdb_options_statistics_get_string(
     crocksdb_options_t* opt);
+extern C_ROCKSDB_LIBRARY_API uint64_t crocksdb_options_statistics_get_ticker_count(
+    crocksdb_options_t* opt, uint32_t ticker_type);
+extern C_ROCKSDB_LIBRARY_API uint64_t crocksdb_options_statistics_get_and_reset_ticker_count(
+    crocksdb_options_t* opt, uint32_t ticker_type);
+extern C_ROCKSDB_LIBRARY_API char*
+crocksdb_options_statistics_get_histogram_string(crocksdb_options_t* opt,
+                                                 uint32_t type);
 
 extern C_ROCKSDB_LIBRARY_API void crocksdb_options_set_max_write_buffer_number(
     crocksdb_options_t*, int);

librocksdb_sys/src/lib.rs

@@ -83,6 +83,55 @@ pub enum DBRecoveryMode {
     SkipAnyCorruptedRecords = 3,
 }
 
+#[derive(Copy, Clone)]
+#[repr(C)]
+pub enum DBStatisticsTickerType {
+    BlockCacheMiss = 0, // total block cache miss
+    BlockCacheHit = 1, // total block cache hit
+    BlockCacheIndexMiss = 4, // times cache miss when accessing index block from block cache
+    BlockCacheIndexHit = 5,
+    BlockCacheFilterMiss = 9, // times cache miss when accessing filter block from block cache
+    BlockCacheFilterHit = 10,
+    BloomFilterUseful = 20, // times bloom filter has avoided file reads
+    MemtableHit = 25,
+    MemtableMiss = 26,
+    GetHitL0 = 27, // Get() queries served by L0
+    GetHitL1 = 28, // Get() queries served by L1
+    GetHitL2AndUp = 29, // Get() queries served by L2 and up
+    NumberKeysWritten = 35, // number of keys written to the database via the Put and Write call's
+    NumberKeysRead = 36, // number of keys read
+    BytesWritten = 38, // the number of uncompressed bytes read from DB::Put, DB::Delete,
+    // DB::Merge and DB::Write
+    BytesRead = 39, // the number of uncompressed bytes read from DB::Get()
+    NumberDbSeek = 40, // the number of calls to seek/next/prev
+    NumberDbNext = 41,
+    NumberDbPrev = 42,
+    NumberDbSeekFound = 43, // the number of calls to seek/next/prev that returned data
+    NumberDbNextFound = 44,
+    NumberDbPrevFound = 45,
+    IterBytesRead = 46, // the number of uncompressed bytes read from an iterator, include size of
+    // key and value
+    StallMicros = 53, // writer has to wait for compaction or flush to finish
+    NoIterators = 56, // number of iterators currently open
+    BloomFilterPrefixChecked = 62, // number of times bloom was checked before creating iterator
+    // on a file
+    BloomFilterPrefixUseful = 63, // number of times the check was useful in avoiding iterator
+    // creating
+    WalFileSynced = 70, // number of times WAL sync is done
+    WalFileBytes = 71, // number of bytes written to WAL
+    CompactReadBytes = 76, // bytes read during compaction
+    CompactWriteBytes = 77, // bytes written during compaction
+    FlushWriteBytes = 78, // bytes written during flush
+}
+
+#[derive(Copy, Clone)]
+#[repr(C)]
+pub enum DBStatisticsHistogramType {
+    DbGetMicros = 0,
+    DbWriteMicros = 1,
+    DbSeekMicros = 19,
+}
+
 pub fn error_message(ptr: *mut c_char) -> String {
     let c_str = unsafe { CStr::from_ptr(ptr) };
     let s = format!("{}", c_str.to_string_lossy());
@@ -191,6 +240,15 @@ extern "C" {
     pub fn crocksdb_options_set_wal_recovery_mode(options: *mut DBOptions, mode: DBRecoveryMode);
     pub fn crocksdb_options_enable_statistics(options: *mut DBOptions);
     pub fn crocksdb_options_statistics_get_string(options: *mut DBOptions) -> *const c_char;
+    pub fn crocksdb_options_statistics_get_ticker_count(options: *mut DBOptions,
+                                                        ticker_type: DBStatisticsTickerType)
+                                                        -> u64;
+    pub fn crocksdb_options_statistics_get_and_reset_ticker_count(options: *mut DBOptions,
+                                                        ticker_type: DBStatisticsTickerType)
+                                                        -> u64;
+    pub fn crocksdb_options_statistics_get_histogram_string(options: *mut DBOptions,
+                                                            hist_type: DBStatisticsHistogramType)
+                                                            -> *const c_char;
     pub fn crocksdb_options_set_stats_dump_period_sec(options: *mut DBOptions, v: usize);
     pub fn crocksdb_options_set_num_levels(options: *mut DBOptions, v: c_int);
     pub fn crocksdb_options_set_db_log_dir(options: *mut DBOptions, path: *const c_char);

src/external_file.rs

-// Copyright 2016 PingCAP, Inc.
+// Copyright 2017 PingCAP, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

src/rocksdb.rs

@@ -13,7 +13,8 @@
 // limitations under the License.
 //
 
-use crocksdb_ffi::{self, DBWriteBatch, DBCFHandle, DBInstance, DBBackupEngine};
+use crocksdb_ffi::{self, DBWriteBatch, DBCFHandle, DBInstance, DBBackupEngine,
+                   DBStatisticsTickerType, DBStatisticsHistogramType};
 use libc::{self, c_int, c_void, size_t};
 use rocksdb_options::{Options, ReadOptions, UnsafeSnap, WriteOptions, FlushOptions,
                       RestoreOptions, IngestExternalFileOptions};
@@ -857,6 +858,22 @@ impl DB {
         self.opts.get_statistics()
     }
 
+    pub fn get_statistics_ticker_count(&self, ticker_type: DBStatisticsTickerType) -> u64 {
+        self.opts.get_statistics_ticker_count(ticker_type)
+    }
+
+    pub fn get_and_reset_statistics_ticker_count(&self,
+                                                 ticker_type: DBStatisticsTickerType)
+                                                 -> u64 {
+        self.opts.get_and_reset_statistics_ticker_count(ticker_type)
+    }
+
+    pub fn get_statistics_histogram_string(&self,
+                                           hist_type: DBStatisticsHistogramType)
+                                           -> Option<String> {
+        self.opts.get_statistics_histogram_string(hist_type)
+    }
+
     pub fn get_options(&self) -> &Options {
         &self.opts
     }

src/rocksdb_options.rs

@@ -18,7 +18,7 @@ use comparator::{self, ComparatorCallback, compare_callback};
 
 use crocksdb_ffi::{self, DBOptions, DBWriteOptions, DBBlockBasedTableOptions, DBReadOptions,
                    DBRestoreOptions, DBCompressionType, DBRecoveryMode, DBSnapshot, DBInstance,
-                   DBFlushOptions};
+                   DBFlushOptions, DBStatisticsTickerType, DBStatisticsHistogramType};
 use libc::{self, c_int, size_t, c_void};
 use merge_operator::{self, MergeOperatorCallback, full_merge_callback, partial_merge_callback};
 use merge_operator::MergeFn;
@@ -525,6 +525,38 @@ impl Options {
         }
     }
 
+    pub fn get_statistics_ticker_count(&self, ticker_type: DBStatisticsTickerType) -> u64 {
+        unsafe {
+            crocksdb_ffi::crocksdb_options_statistics_get_ticker_count(self.inner, ticker_type)
+        }
+    }
+
+    pub fn get_and_reset_statistics_ticker_count(&self,
+                                                 ticker_type: DBStatisticsTickerType)
+                                                 -> u64 {
+        unsafe {
+            crocksdb_ffi::crocksdb_options_statistics_get_and_reset_ticker_count(self.inner,
+                                                                                 ticker_type)
+        }
+    }
+
+    pub fn get_statistics_histogram_string(&self,
+                                           hist_type: DBStatisticsHistogramType)
+                                           -> Option<String> {
+        unsafe {
+            let value = crocksdb_ffi::crocksdb_options_statistics_get_histogram_string(self.inner,
+                                                                                       hist_type);
+
+            if value.is_null() {
+                return None;
+            }
+
+            let s = CStr::from_ptr(value).to_str().unwrap().to_owned();
+            libc::free(value as *mut c_void);
+            Some(s)
+        }
+    }
+
     pub fn get_statistics(&self) -> Option<String> {
         unsafe {
             let value = crocksdb_ffi::crocksdb_options_statistics_get_string(self.inner);
@@ -732,26 +764,3 @@ impl Drop for RestoreOptions {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::Options;
-
-    #[test]
-    fn test_set_max_manifest_file_size() {
-        let mut opts = Options::new();
-        let size = 20 * 1024 * 1024;
-        opts.set_max_manifest_file_size(size)
-    }
-
-    #[test]
-    fn test_enable_statistics() {
-        let mut opts = Options::new();
-        opts.enable_statistics();
-        opts.set_stats_dump_period_sec(60);
-        assert!(opts.get_statistics().is_some());
-
-        let opts = Options::new();
-        assert!(opts.get_statistics().is_none());
-    }
-}

src/slice_transform.rs

-// Copyright 2016 PingCAP, Inc.
+// Copyright 2017 PingCAP, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

test/test.rs

@@ -10,3 +10,4 @@ mod test_rocksdb_options;
 mod test_ingest_external_file;
 mod test_slice_transform;
 mod test_prefix_extractor;
+mod test_statistics;

test/test_compact_range.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 use rocksdb::{DB, Options, Range, Writable};
 use tempdir::TempDir;

test/test_compaction_filter.rs

-
+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 use rocksdb::{Writable, DB, CompactionFilter, Options};
 use std::sync::{Arc, RwLock};

test/test_ingest_external_file.rs

-// Copyright 2016 PingCAP, Inc.
+// Copyright 2017 PingCAP, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 // limitations under the License.
 
 use rocksdb::*;
-use tempdir::TempDir;
 
 use std::fs;
+use tempdir::TempDir;
 
 fn gen_sst(opt: &Options, path: &str, data: &[(&[u8], &[u8])]) {
     let _ = fs::remove_file(path);
@@ -43,29 +43,35 @@ fn test_ingest_external_file() {
     let test_sstfile = gen_path.path().join("test_sst_file");
     let test_sstfile_str = test_sstfile.to_str().unwrap();
 
-    gen_sst(db.get_options(), test_sstfile_str, &[(b"k1", b"v1"), (b"k2", b"v2")]);
-    
+    gen_sst(db.get_options(),
+            test_sstfile_str,
+            &[(b"k1", b"v1"), (b"k2", b"v2")]);
+
     let mut ingest_opt = IngestExternalFileOptions::new();
     db.ingest_external_file(&ingest_opt, &[test_sstfile_str]).unwrap();
     assert!(test_sstfile.exists());
     assert_eq!(db.get(b"k1").unwrap().unwrap(), b"v1");
     assert_eq!(db.get(b"k2").unwrap().unwrap(), b"v2");
 
-    gen_sst(&cf_opts, test_sstfile_str, &[(b"k1", b"v3"), (b"k2", b"v4")]);
+    gen_sst(&cf_opts,
+            test_sstfile_str,
+            &[(b"k1", b"v3"), (b"k2", b"v4")]);
     db.ingest_external_file_cf(handle, &ingest_opt, &[test_sstfile_str]).unwrap();
     assert_eq!(db.get_cf(handle, b"k1").unwrap().unwrap(), b"v3");
     assert_eq!(db.get_cf(handle, b"k2").unwrap().unwrap(), b"v4");
 
     let snap = db.snapshot();
 
-    gen_sst(db.get_options(), test_sstfile_str, &[(b"k2", b"v5"), (b"k3", b"v6")]);
+    gen_sst(db.get_options(),
+            test_sstfile_str,
+            &[(b"k2", b"v5"), (b"k3", b"v6")]);
     ingest_opt = ingest_opt.move_files(true);
     db.ingest_external_file_cf(handle, &ingest_opt, &[test_sstfile_str]).unwrap();
-    
+
     assert_eq!(db.get_cf(handle, b"k1").unwrap().unwrap(), b"v3");
     assert_eq!(db.get_cf(handle, b"k2").unwrap().unwrap(), b"v5");
     assert_eq!(db.get_cf(handle, b"k3").unwrap().unwrap(), b"v6");
     assert_eq!(snap.get_cf(handle, b"k1").unwrap().unwrap(), b"v3");
     assert_eq!(snap.get_cf(handle, b"k2").unwrap().unwrap(), b"v4");
     assert!(snap.get_cf(handle, b"k3").unwrap().is_none());
-}
\ No newline at end of file
+}

test/test_iterator.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use rocksdb::*;
 use tempdir::TempDir;
 

test/test_multithreaded.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use rocksdb::{DB, Writable};
 use std::sync::Arc;
 use std::thread;

test/test_prefix_extractor.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 use rocksdb::*;
 use tempdir::TempDir;
 

test/test_rocksdb_options.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 use rocksdb::{DB, Options};
+use rocksdb::crocksdb_ffi::{DBStatisticsHistogramType as HistogramType,
+                            DBStatisticsTickerType as TickerType};
 use tempdir::TempDir;
 
 
@@ -33,3 +47,28 @@ fn test_compaction_readahead_size() {
     let db = DB::open(opts, path.path().to_str().unwrap()).unwrap();
     drop(db);
 }
+
+#[test]
+fn test_set_max_manifest_file_size() {
+    let mut opts = Options::new();
+    let size = 20 * 1024 * 1024;
+    opts.set_max_manifest_file_size(size)
+}
+
+#[test]
+fn test_enable_statistics() {
+    let mut opts = Options::new();
+    opts.enable_statistics();
+    opts.set_stats_dump_period_sec(60);
+    assert!(opts.get_statistics().is_some());
+    assert!(opts.get_statistics_histogram_string(HistogramType::DbSeekMicros).is_some());
+    assert_eq!(opts.get_statistics_ticker_count(TickerType::BlockCacheMiss),
+               0);
+    assert_eq!(opts.get_and_reset_statistics_ticker_count(TickerType::BlockCacheMiss),
+               0);
+    assert_eq!(opts.get_statistics_ticker_count(TickerType::BlockCacheMiss),
+               0);
+
+    let opts = Options::new();
+    assert!(opts.get_statistics().is_none());
+}

test/test_slice_transform.rs

-// Copyright 2016 PingCAP, Inc.
+// Copyright 2017 PingCAP, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,8 +11,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
-
 use rocksdb::{Writable, DB, SliceTransform, Options, SeekKey, BlockBasedOptions};
 use tempdir::TempDir;
 

test/test_statistics.rs

+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rocksdb::*;
+use self::crocksdb_ffi::{DBStatisticsTickerType as TickerType,
+                         DBStatisticsHistogramType as HistogramType};
+use tempdir::TempDir;
+
+#[test]
+fn test_db_statistics() {
+    let path = TempDir::new("_rust_rocksdb_statistics").expect("");
+    let mut opts = Options::new();
+    opts.create_if_missing(true);
+    opts.enable_statistics();
+    let db = DB::open(opts, path.path().to_str().unwrap()).unwrap();
+    let wopts = WriteOptions::new();
+
+    db.put_opt(b"k0", b"a", &wopts).unwrap();
+    db.put_opt(b"k1", b"b", &wopts).unwrap();
+    db.put_opt(b"k2", b"c", &wopts).unwrap();
+    db.flush(true /* sync */).unwrap(); // flush memtable to sst file.
+    assert_eq!(db.get(b"k0").unwrap().unwrap(), b"a");
+    assert_eq!(db.get(b"k1").unwrap().unwrap(), b"b");
+    assert_eq!(db.get(b"k2").unwrap().unwrap(), b"c");
+
+    assert!(db.get_statistics_ticker_count(TickerType::BlockCacheHit) > 0);
+    assert!(db.get_and_reset_statistics_ticker_count(TickerType::BlockCacheHit) > 0);
+    assert_eq!(db.get_statistics_ticker_count(TickerType::BlockCacheHit), 0);
+    assert!(db.get_statistics_histogram_string(HistogramType::DbGetMicros).is_some());
+}