initial checkpoint for some optimization options

src/ffi.rs

@@ -62,6 +62,8 @@ extern {
         options: RocksDBOptions, bits: u64);
     pub fn rocksdb_options_set_max_write_buffer_number(
         options: RocksDBOptions, bufno: c_int);
+    pub fn rocksdb_options_set_max_write_buffer_number_to_merge(
+        options: RocksDBOptions, bufno: c_int);
     pub fn rocksdb_options_set_min_write_buffer_number_to_merge(
         options: RocksDBOptions, bufno: c_int);
     pub fn rocksdb_options_set_level0_file_num_compaction_trigger(
@@ -92,6 +94,8 @@ extern {
         options: RocksDBOptions, max_bg_flushes: c_int);
     pub fn rocksdb_options_set_filter_deletes(
         options: RocksDBOptions, v: u8);
+    pub fn rocksdb_options_set_disable_auto_compactions(
+        options: RocksDBOptions, v: u8);
     //pub fn rocksdb_compactionfilter_create() -> RocksDBCompactionFilter;
     pub fn rocksdb_filterpolicy_create_bloom(
         bits_per_key: c_int) -> RocksDBFilterPolicy;

src/lib.rs

@@ -3,6 +3,11 @@
 #![allow(dead_code)]
 
 pub use ffi as rocksdb_ffi;
+pub use ffi::{
+    RocksDBUniversalCompactionStyle,
+    RocksDBCompactionStyle,
+    RocksDBCompressionType,
+};
 pub use rocksdb::{
     RocksDB,
     MergeOperands,

src/main.rs

 extern crate rocksdb;
 extern crate test;
 use rocksdb::{RocksDBOptions, RocksDB, MergeOperands};
+use rocksdb::RocksDBCompactionStyle::RocksDBUniversalCompaction;
 use test::Bencher;
 
 #[allow(dead_code)]
@@ -23,6 +24,7 @@ fn main() {
     db.close();
 
     custom_merge();
+    optimized();
 }
 
 #[allow(dead_code)]
@@ -58,6 +60,46 @@ fn custom_merge() {
     RocksDB::destroy(opts, path).is_ok();
 }
 
+#[allow(dead_code)]
+fn optimized() {
+    let path = "_rust_rocksdb_optimizetest";
+    let opts = RocksDBOptions::new();
+    opts.create_if_missing(true);
+    opts.set_block_size(524288);
+    opts.set_max_open_files(10000);
+    opts.set_use_fsync(false);
+    opts.set_bytes_per_sync(8388608);
+    opts.set_disable_data_sync(false);
+    opts.set_cache_size(8589934592);
+    opts.set_table_cache_num_shard_bits(6);
+    opts.set_max_write_buffer_number(32);
+    opts.set_write_buffer_size(536870912);
+    opts.set_target_file_size_base(1073741824);
+    opts.set_min_write_buffer_number_to_merge(4);
+    opts.set_level_zero_stop_writes_trigger(2000);
+    opts.set_level_zero_slowdown_writes_trigger(0);
+    //opts.set_memtable_config(newSkipListMemTableConfig());
+    opts.set_compaction_style(RocksDBUniversalCompaction);
+    opts.set_max_background_compactions(4);
+    opts.set_max_background_flushes(4);
+    opts.set_filter_deletes(false);
+    opts.set_disable_auto_compaction(true);
+    //opts.set_filter(filter);
+
+    opts.add_merge_operator("test operator", concat_merge);
+    let db = RocksDB::open(opts, path).unwrap();
+    let p = db.put(b"k1", b"a");
+    db.merge(b"k1", b"b");
+    db.merge(b"k1", b"c");
+    db.merge(b"k1", b"d");
+    db.merge(b"k1", b"efg");
+    let m = db.merge(b"k1", b"h");
+    let r = db.get(b"k1");
+    assert!(r.unwrap().to_utf8().unwrap() == "abcdefgh");
+    db.close();
+    RocksDB::destroy(opts, path).is_ok();
+}
+
 #[allow(dead_code)]
 #[bench]
 fn writes(b: &mut Bencher) {

src/rocksdb.rs

@@ -71,6 +71,145 @@ impl RocksDBOptions {
             rocksdb_ffi::rocksdb_options_set_merge_operator(self.inner, mo);
         }
     }
+
+    /* block based table options
+    pub fn set_block_size(&self, size: u64) {
+        unsafe {
+            rocksdb_ffi::rocksdb_block_based_options_set_block_size(self.table_options, size);
+        }
+    }
+
+    pub fn set_cache_size(&self, cache_size: u64) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set(self.inner, );
+        }
+    }
+
+    pub fn set_memtable_config(&self,newSkipListMemTableConfig()) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set(self.inner, );
+        }
+    }
+
+    pub fn set_filter(&self, filter: RocksDBFilterPolicy) {
+        unsafe {
+            rocksdb_ffi::rocksdb_block_based_options_set_filter_policy(self.inner, filter);
+        }
+    }
+
+    */
+
+    pub fn set_max_open_files(&self, nfiles: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_max_open_files(self.inner, nfiles);
+        }
+    }
+
+    pub fn set_use_fsync(&self, useit: bool) {
+        unsafe {
+            match useit {
+                true => rocksdb_ffi::rocksdb_options_set_use_fsync(
+                    self.inner, 1),
+                false => rocksdb_ffi::rocksdb_options_set_use_fsync(
+                    self.inner, 0),
+            }
+        }
+    }
+
+    pub fn set_bytes_per_sync(&self, nbytes: u64) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_bytes_per_sync(self.inner, nbytes);
+        }
+    }
+
+    pub fn set_disable_data_sync(&self, disable: bool) {
+        unsafe {
+            match disable {
+                true =>
+                    rocksdb_ffi::rocksdb_options_set_disable_data_sync(self.inner, 1),
+                false =>
+                    rocksdb_ffi::rocksdb_options_set_disable_data_sync(self.inner, 0),
+            }
+        }
+    }
+
+    pub fn set_table_cache_num_shard_bits(&self, nbits: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_table_cache_numshardbits(self.inner, nbits);
+        }
+    }
+
+    pub fn set_min_write_buffer_number(&self, nbuf: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge(self.inner, nbuf);
+        }
+    }
+
+    pub fn set_max_write_buffer_number(&self, nbuf: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_max_write_buffer_number_to_merge(self.inner, nbuf);
+        }
+    }
+
+    pub fn set_write_buffer_size(&self, size: size_t) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_write_buffer_size(self.inner, size);
+        }
+    }
+
+    pub fn set_target_file_size_base(&self, size: u64) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_target_file_size_base(self.inner, size);
+        }
+    }
+
+    pub fn set_min_write_buffer_number_to_merge(&self, to_merge: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge(self.inner, to_merge);
+        }
+    }
+
+    pub fn set_level_zero_slowdown_writes_trigger(&self, n: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_level0_slowdown_writes_trigger(self.inner, n);
+        }
+    }
+
+    pub fn set_level_zero_stop_writes_trigger(&self, n: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_level0_stop_writes_trigger(self.inner, n);
+        }
+    }
+
+    pub fn set_compaction_style(&self, style: rocksdb_ffi::RocksDBCompactionStyle) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_compaction_style(self.inner, style);
+        }
+    }
+
+    pub fn set_max_background_compactions(&self, n: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_max_background_compactions(self.inner, n);
+        }
+    }
+
+    pub fn set_max_background_flushes(&self, n: c_int) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_max_background_flushes(self.inner, n);
+        }
+    }
+
+    pub fn set_filter_deletes(&self, filter: bool) { // to u8
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_filter_deletes(self.inner, filter);
+        }
+    }
+
+    pub fn set_disable_auto_compactions(&self, disable: bool) {
+        unsafe {
+            rocksdb_ffi::rocksdb_options_set_disable_auto_compactions(self.inner, disable);
+        }
+    }
 }
 
 pub struct RocksDB {
@@ -448,6 +587,11 @@ extern "C" fn full_merge_callback(
         let oldval = from_buf_len(existing_value as *const u8, existing_value_len as uint);
         let mut result = (cb.merge_fn)(key.as_bytes(), Some(oldval.as_bytes()), operands);
         result.shrink_to_fit();
+        /*
+        let ptr = result.as_ptr();
+        mem::forget(result);
+        ptr as *const c_char
+        */
         //TODO(tan) investigate zero-copy techniques to improve performance
         let buf = libc::malloc(result.len() as size_t);
         assert!(buf.is_not_null());
@@ -517,7 +661,7 @@ fn mergetest() {
                     println!("did not read valid utf-8 out of the db"),
             }
         }).on_absent( || { println!("value not present!") })
-          .on_error( |e| { println!("error reading value")}); //: {}", e) });
+          .on_error( |e| { println!("error reading value")}); //: {", e) });
 
         assert!(m.is_ok());
         let r: RocksDBResult<RocksDBVector, String> = db.get(b"k1");