Merge pull request #2 from spacejam/tan_optimization

Tan optimization

README.md

@@ -67,6 +67,41 @@ fn main() {
 }
 ```
 
+###### Apply Some Tunings
+Please read [the official tuning guide](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide), and most importantly, measure performance under realistic workloads with realistic hardware.
+```rust
+use rocksdb::{RocksDBOptions, RocksDB, new_bloom_filter};
+use rocksdb::RocksDBCompactionStyle::RocksDBUniversalCompaction;
+
+fn tuned_for_somebody_elses_disk() -> RocksDB {
+    let path = "_rust_rocksdb_optimizetest";
+    let opts = RocksDBOptions::new();
+    opts.create_if_missing(true);
+    opts.set_block_size(524288);
+    opts.set_max_open_files(10000);
+    opts.set_use_fsync(false);
+    opts.set_bytes_per_sync(8388608);
+    opts.set_disable_data_sync(false);
+    opts.set_block_cache_size_mb(1024);
+    opts.set_table_cache_num_shard_bits(6);
+    opts.set_max_write_buffer_number(32);
+    opts.set_write_buffer_size(536870912);
+    opts.set_target_file_size_base(1073741824);
+    opts.set_min_write_buffer_number_to_merge(4);
+    opts.set_level_zero_stop_writes_trigger(2000);
+    opts.set_level_zero_slowdown_writes_trigger(0);
+    opts.set_compaction_style(RocksDBUniversalCompaction);
+    opts.set_max_background_compactions(4);
+    opts.set_max_background_flushes(4);
+    opts.set_filter_deletes(false);
+    opts.set_disable_auto_compactions(true);
+
+    let filter = new_bloom_filter(10);
+    opts.set_filter(filter);
+
+    RocksDB::open(opts, path).unwrap()
+}
+```
 
 ### status
   - [x] basic open/put/get/delete/close

src/ffi.rs

@@ -10,12 +10,20 @@ pub struct RocksDBWriteOptions(pub *const c_void);
 #[repr(C)]
 pub struct RocksDBReadOptions(pub *const c_void);
 #[repr(C)]
-pub struct RocksDBCompactionFilter(pub *const c_void);
-#[repr(C)]
 pub struct RocksDBMergeOperator(pub *const c_void);
 #[repr(C)]
+pub struct RocksDBBlockBasedTableOptions(pub *const c_void);
+#[repr(C)]
+pub struct RocksDBCache(pub *const c_void);
+#[repr(C)]
 pub struct RocksDBFilterPolicy(pub *const c_void);
 
+pub fn new_bloom_filter(bits: c_int) -> RocksDBFilterPolicy {
+    unsafe {
+        rocksdb_filterpolicy_create_bloom(bits)
+    }
+}
+
 #[repr(C)]
 pub enum RocksDBCompressionType {
     RocksDBNoCompression     = 0,
@@ -42,12 +50,41 @@ pub enum RocksDBUniversalCompactionStyle {
 #[link(name = "rocksdb")]
 extern {
     pub fn rocksdb_options_create() -> RocksDBOptions;
+    pub fn rocksdb_cache_create_lru(capacity: size_t) -> RocksDBCache;
+    pub fn rocksdb_cache_destroy(cache: RocksDBCache);
+    pub fn rocksdb_block_based_options_create() -> RocksDBBlockBasedTableOptions;
+    pub fn rocksdb_block_based_options_destroy(
+        block_options: RocksDBBlockBasedTableOptions);
+    pub fn rocksdb_block_based_options_set_block_size(
+        block_options: RocksDBBlockBasedTableOptions,
+        block_size: size_t);
+    pub fn rocksdb_block_based_options_set_block_size_deviation(
+        block_options: RocksDBBlockBasedTableOptions,
+        block_size_deviation: c_int);
+    pub fn rocksdb_block_based_options_set_block_restart_interval(
+        block_options: RocksDBBlockBasedTableOptions,
+        block_restart_interval: c_int);
+    pub fn rocksdb_block_based_options_set_filter_policy(
+        block_options: RocksDBBlockBasedTableOptions,
+        filter_policy: RocksDBFilterPolicy);
+    pub fn rocksdb_block_based_options_set_no_block_cache(
+        block_options: RocksDBBlockBasedTableOptions, no_block_cache: bool);
+    pub fn rocksdb_block_based_options_set_block_cache(
+        block_options: RocksDBBlockBasedTableOptions, block_cache: RocksDBCache);
+    pub fn rocksdb_block_based_options_set_block_cache_compressed(
+        block_options: RocksDBBlockBasedTableOptions,
+        block_cache_compressed: RocksDBCache);
+    pub fn rocksdb_block_based_options_set_whole_key_filtering(
+        ck_options: RocksDBBlockBasedTableOptions, doit: bool);
+    pub fn rocksdb_options_set_block_based_table_factory(
+        options: RocksDBOptions,
+        block_options: RocksDBBlockBasedTableOptions);
     pub fn rocksdb_options_increase_parallelism(
         options: RocksDBOptions, threads: c_int);
     pub fn rocksdb_options_optimize_level_style_compaction(
         options: RocksDBOptions, memtable_memory_budget: c_int);
     pub fn rocksdb_options_set_create_if_missing(
-        options: RocksDBOptions, v: c_int);
+        options: RocksDBOptions, v: bool);
     pub fn rocksdb_options_set_max_open_files(
         options: RocksDBOptions, files: c_int);
     pub fn rocksdb_options_set_use_fsync(
@@ -59,7 +96,7 @@ extern {
     pub fn rocksdb_options_optimize_for_point_lookup(
         options: RocksDBOptions, block_cache_size_mb: u64);
     pub fn rocksdb_options_set_table_cache_numshardbits(
-        options: RocksDBOptions, bits: u64);
+        options: RocksDBOptions, bits: c_int);
     pub fn rocksdb_options_set_max_write_buffer_number(
         options: RocksDBOptions, bufno: c_int);
     pub fn rocksdb_options_set_min_write_buffer_number_to_merge(
@@ -91,8 +128,9 @@ extern {
     pub fn rocksdb_options_set_max_background_flushes(
         options: RocksDBOptions, max_bg_flushes: c_int);
     pub fn rocksdb_options_set_filter_deletes(
-        options: RocksDBOptions, v: u8);
-    //pub fn rocksdb_compactionfilter_create() -> RocksDBCompactionFilter;
+        options: RocksDBOptions, v: bool);
+    pub fn rocksdb_options_set_disable_auto_compactions(
+        options: RocksDBOptions, v: c_int);
     pub fn rocksdb_filterpolicy_create_bloom(
         bits_per_key: c_int) -> RocksDBFilterPolicy;
     pub fn rocksdb_open(options: RocksDBOptions,
@@ -152,7 +190,7 @@ fn internal() {
 
         rocksdb_options_increase_parallelism(opts, 0);
         rocksdb_options_optimize_level_style_compaction(opts, 0);
-        rocksdb_options_set_create_if_missing(opts, 1);
+        rocksdb_options_set_create_if_missing(opts, true);
 
         let rustpath = "_rust_rocksdb_internaltest";
         let cpath = rustpath.to_c_str();

src/lib.rs

@@ -3,6 +3,12 @@
 #![allow(dead_code)]
 
 pub use ffi as rocksdb_ffi;
+pub use ffi::{
+    new_bloom_filter,
+    RocksDBUniversalCompactionStyle,
+    RocksDBCompactionStyle,
+    RocksDBCompressionType,
+};
 pub use rocksdb::{
     RocksDB,
     MergeOperands,

src/main.rs

 extern crate rocksdb;
 extern crate test;
-use rocksdb::{RocksDBOptions, RocksDB, MergeOperands};
+use rocksdb::{RocksDBOptions, RocksDB, MergeOperands, new_bloom_filter};
+use rocksdb::RocksDBCompactionStyle::RocksDBUniversalCompaction;
 use test::Bencher;
 
 #[allow(dead_code)]
@@ -58,10 +59,40 @@ fn custom_merge() {
     RocksDB::destroy(opts, path).is_ok();
 }
 
+#[allow(dead_code)]
+fn tuned_for_somebody_elses_disk() -> RocksDB {
+    let path = "_rust_rocksdb_optimizetest";
+    let opts = RocksDBOptions::new();
+    opts.create_if_missing(true);
+    opts.set_block_size(524288);
+    opts.set_max_open_files(10000);
+    opts.set_use_fsync(false);
+    opts.set_bytes_per_sync(8388608);
+    opts.set_disable_data_sync(false);
+    opts.set_block_cache_size_mb(1024);
+    opts.set_table_cache_num_shard_bits(6);
+    opts.set_max_write_buffer_number(32);
+    opts.set_write_buffer_size(536870912);
+    opts.set_target_file_size_base(1073741824);
+    opts.set_min_write_buffer_number_to_merge(4);
+    opts.set_level_zero_stop_writes_trigger(2000);
+    opts.set_level_zero_slowdown_writes_trigger(0);
+    opts.set_compaction_style(RocksDBUniversalCompaction);
+    opts.set_max_background_compactions(4);
+    opts.set_max_background_flushes(4);
+    opts.set_filter_deletes(false);
+    opts.set_disable_auto_compactions(true);
+
+    let filter = new_bloom_filter(10);
+    opts.set_filter(filter);
+
+    RocksDB::open(opts, path).unwrap()
+}
+
 #[allow(dead_code)]
 #[bench]
 fn writes(b: &mut Bencher) {
-    let db = RocksDB::open_default("testdb").unwrap();
+    let db = tuned_for_somebody_elses_disk();
     let mut i = 0 as u64;
     b.iter(|| {
         db.put(i.to_string().as_bytes(), b"v1111");
@@ -73,7 +104,7 @@ fn writes(b: &mut Bencher) {
 #[allow(dead_code)]
 #[bench]
 fn reads(b: &mut Bencher) {
-    let db = RocksDB::open_default("testdb").unwrap();
+    let db = tuned_for_somebody_elses_disk();
     let mut i = 0 as u64;
     b.iter(|| {
         db.get(i.to_string().as_bytes()).on_error( |e| {