Upgrade RocksDB to 6.4.x (#336)

Upgrade RocksDB to 6.4.x. Also updating titan master.
Signed-off-by: Yi Wu <yiwu@pingcap.com>

.gitmodules

 [submodule "rocksdb"]
 	path = librocksdb_sys/rocksdb
 	url = https://github.com/pingcap/rocksdb.git
-	branch = tikv-3.0
+	branch = 6.4.tikv
 [submodule "titan"]
 	path = librocksdb_sys/libtitan_sys/titan
 	url = https://github.com/pingcap/titan.git
+	branch = master
+

.travis.yml

@@ -19,9 +19,7 @@ env:
   global:
   - RUST_BACKTRACE=1
 
-cache:
-  directories:
-  - target
+cache: false
 
 before_script:
   - rustup component add rustfmt-preview

librocksdb_sys/build.rs

@@ -140,6 +140,10 @@ fn build_rocksdb() -> Build {
     build.include(cur_dir.join("libtitan_sys").join("titan").join("include"));
     build.include(cur_dir.join("libtitan_sys").join("titan"));
 
+    // Adding rocksdb specific compile macros.
+    // TODO: should make sure crocksdb compile options is the same as rocksdb and titan.
+    build.define("ROCKSDB_SUPPORT_THREAD_LOCAL", None);
+
     println!("cargo:rustc-link-lib=static=rocksdb");
     println!("cargo:rustc-link-lib=static=titan");
     println!("cargo:rustc-link-lib=static=z");

librocksdb_sys/crocksdb/c.cc

@@ -41,8 +41,8 @@
 
 #include "db/column_family.h"
 #include "table/sst_file_writer_collectors.h"
+#include "table/block_based/block_based_table_factory.h"
 #include "table/table_reader.h"
-#include "table/block_based_table_factory.h"
 #include "util/file_reader_writer.h"
 #include "util/coding.h"
 
@@ -3899,11 +3899,6 @@ void crocksdb_fifo_compaction_options_set_max_table_files_size(
   fifo_opts->rep.max_table_files_size = size;
 }
 
-void crocksdb_fifo_compaction_options_set_ttl(
-    crocksdb_fifo_compaction_options_t* fifo_opts, uint64_t ttl) {
-  fifo_opts->rep.ttl = ttl;
-}
-
 void crocksdb_fifo_compaction_options_set_allow_compaction(
     crocksdb_fifo_compaction_options_t* fifo_opts, bool allow_compaction) {
   fifo_opts->rep.allow_compaction = allow_compaction;
@@ -5247,9 +5242,9 @@ void ctitandb_options_set_max_background_gc(ctitandb_options_t* options,
   options->rep.max_background_gc = size;
 }
 
-void ctitandb_options_set_purge_obsolete_files_period(ctitandb_options_t* options,
-                                                      unsigned int period) {
-  options->rep.purge_obsolete_files_period = period;
+void ctitandb_options_set_purge_obsolete_files_period_sec(
+    ctitandb_options_t* options, unsigned int period) {
+  options->rep.purge_obsolete_files_period_sec = period;
 }
 
 void ctitandb_options_set_blob_cache(ctitandb_options_t* options,

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -1563,9 +1563,6 @@ extern C_ROCKSDB_LIBRARY_API void
 crocksdb_fifo_compaction_options_set_max_table_files_size(
     crocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size);
 extern C_ROCKSDB_LIBRARY_API void
-crocksdb_fifo_compaction_options_set_ttl(
-    crocksdb_fifo_compaction_options_t* fifo_opts, uint64_t ttl);
-extern C_ROCKSDB_LIBRARY_API void
 crocksdb_fifo_compaction_options_set_allow_compaction(
     crocksdb_fifo_compaction_options_t* fifo_opts, bool allow_compaction);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_fifo_compaction_options_destroy(
@@ -2102,7 +2099,8 @@ ctitandb_options_set_merge_small_file_threshold(ctitandb_options_t* options,
 extern C_ROCKSDB_LIBRARY_API void ctitandb_options_set_max_background_gc(
     ctitandb_options_t* options, int32_t size);
 
-extern C_ROCKSDB_LIBRARY_API void ctitandb_options_set_purge_obsolete_files_period(
+extern C_ROCKSDB_LIBRARY_API void
+ctitandb_options_set_purge_obsolete_files_period_sec(
     ctitandb_options_t* options, unsigned int period);
 
 extern C_ROCKSDB_LIBRARY_API void ctitandb_options_set_blob_cache(

titan @ 974e5c6a

-Subproject commit 9cd140f9cbd41ba7b1d477e11a8f1c40a1c23e01
+Subproject commit 974e5c6a0fdf429d2b39e154e5897beeed80aa61

rocksdb @ 5479d051

-Subproject commit a29144f8e992c36f429d74603c43cb22fbead616
+Subproject commit 5479d0516c5b1100fa8b7120d5bb8632b329a9bb

librocksdb_sys/src/generated.rs

@@ -27,120 +27,128 @@ pub enum DBStatisticsTickerType {
     BloomFilterUseful = 20,
     BloomFilterFullPositive = 21,
     BloomFilterFullTruePositive = 22,
-    PersistentCacheHit = 23,
-    PersistentCacheMiss = 24,
-    SimBlockCacheHit = 25,
-    SimBlockCacheMiss = 26,
-    MemtableHit = 27,
-    MemtableMiss = 28,
-    GetHitL0 = 29,
-    GetHitL1 = 30,
-    GetHitL2AndUp = 31,
-    CompactionKeyDropNewerEntry = 32,
-    CompactionKeyDropObsolete = 33,
-    CompactionKeyDropRangeDel = 34,
-    CompactionKeyDropUser = 35,
-    CompactionRangeDelDropObsolete = 36,
-    CompactionOptimizedDelDropObsolete = 37,
-    CompactionCancelled = 38,
-    NumberKeysWritten = 39,
-    NumberKeysRead = 40,
-    NumberKeysUpdated = 41,
-    BytesWritten = 42,
-    BytesRead = 43,
-    NumberDbSeek = 44,
-    NumberDbNext = 45,
-    NumberDbPrev = 46,
-    NumberDbSeekFound = 47,
-    NumberDbNextFound = 48,
-    NumberDbPrevFound = 49,
-    IterBytesRead = 50,
-    NoFileCloses = 51,
-    NoFileOpens = 52,
-    NoFileErrors = 53,
-    StallL0SlowdownMicros = 54,
-    StallMemtableCompactionMicros = 55,
-    StallL0NumFilesMicros = 56,
-    StallMicros = 57,
-    DbMutexWaitMicros = 58,
-    RateLimitDelayMillis = 59,
-    NoIterators = 60,
-    NumberMultigetCalls = 61,
-    NumberMultigetKeysRead = 62,
-    NumberMultigetBytesRead = 63,
-    NumberFilteredDeletes = 64,
-    NumberMergeFailures = 65,
-    BloomFilterPrefixChecked = 66,
-    BloomFilterPrefixUseful = 67,
-    NumberOfReseeksInIteration = 68,
-    GetUpdatesSinceCalls = 69,
-    BlockCacheCompressedMiss = 70,
-    BlockCacheCompressedHit = 71,
-    BlockCacheCompressedAdd = 72,
-    BlockCacheCompressedAddFailures = 73,
-    WalFileSynced = 74,
-    WalFileBytes = 75,
-    WriteDoneBySelf = 76,
-    WriteDoneByOther = 77,
-    WriteTimedout = 78,
-    WriteWithWal = 79,
-    CompactReadBytes = 80,
-    CompactWriteBytes = 81,
-    FlushWriteBytes = 82,
-    NumberDirectLoadTableProperties = 83,
-    NumberSuperversionAcquires = 84,
-    NumberSuperversionReleases = 85,
-    NumberSuperversionCleanups = 86,
-    NumberBlockCompressed = 87,
-    NumberBlockDecompressed = 88,
-    NumberBlockNotCompressed = 89,
-    MergeOperationTotalTime = 90,
-    FilterOperationTotalTime = 91,
-    RowCacheHit = 92,
-    RowCacheMiss = 93,
-    ReadAmpEstimateUsefulBytes = 94,
-    ReadAmpTotalReadBytes = 95,
-    NumberRateLimiterDrains = 96,
-    NumberIterSkip = 97,
-    BlobDbNumPut = 98,
-    BlobDbNumWrite = 99,
-    BlobDbNumGet = 100,
-    BlobDbNumMultiget = 101,
-    BlobDbNumSeek = 102,
-    BlobDbNumNext = 103,
-    BlobDbNumPrev = 104,
-    BlobDbNumKeysWritten = 105,
-    BlobDbNumKeysRead = 106,
-    BlobDbBytesWritten = 107,
-    BlobDbBytesRead = 108,
-    BlobDbWriteInlined = 109,
-    BlobDbWriteInlinedTtl = 110,
-    BlobDbWriteBlob = 111,
-    BlobDbWriteBlobTtl = 112,
-    BlobDbBlobFileBytesWritten = 113,
-    BlobDbBlobFileBytesRead = 114,
-    BlobDbBlobFileSynced = 115,
-    BlobDbBlobIndexExpiredCount = 116,
-    BlobDbBlobIndexExpiredSize = 117,
-    BlobDbBlobIndexEvictedCount = 118,
-    BlobDbBlobIndexEvictedSize = 119,
-    BlobDbGcNumFiles = 120,
-    BlobDbGcNumNewFiles = 121,
-    BlobDbGcFailures = 122,
-    BlobDbGcNumKeysOverwritten = 123,
-    BlobDbGcNumKeysExpired = 124,
-    BlobDbGcNumKeysRelocated = 125,
-    BlobDbGcBytesOverwritten = 126,
-    BlobDbGcBytesExpired = 127,
-    BlobDbGcBytesRelocated = 128,
-    BlobDbFifoNumFilesEvicted = 129,
-    BlobDbFifoNumKeysEvicted = 130,
-    BlobDbFifoBytesEvicted = 131,
-    TxnPrepareMutexOverhead = 132,
-    TxnOldCommitMapMutexOverhead = 133,
-    TxnDuplicateKeyOverhead = 134,
-    TxnSnapshotMutexOverhead = 135,
-    NumberMultigetKeysFound = 136,
+    BloomFilterMicros = 23,
+    PersistentCacheHit = 24,
+    PersistentCacheMiss = 25,
+    SimBlockCacheHit = 26,
+    SimBlockCacheMiss = 27,
+    MemtableHit = 28,
+    MemtableMiss = 29,
+    GetHitL0 = 30,
+    GetHitL1 = 31,
+    GetHitL2AndUp = 32,
+    CompactionKeyDropNewerEntry = 33,
+    CompactionKeyDropObsolete = 34,
+    CompactionKeyDropRangeDel = 35,
+    CompactionKeyDropUser = 36,
+    CompactionRangeDelDropObsolete = 37,
+    CompactionOptimizedDelDropObsolete = 38,
+    CompactionCancelled = 39,
+    NumberKeysWritten = 40,
+    NumberKeysRead = 41,
+    NumberKeysUpdated = 42,
+    BytesWritten = 43,
+    BytesRead = 44,
+    NumberDbSeek = 45,
+    NumberDbNext = 46,
+    NumberDbPrev = 47,
+    NumberDbSeekFound = 48,
+    NumberDbNextFound = 49,
+    NumberDbPrevFound = 50,
+    IterBytesRead = 51,
+    NoFileCloses = 52,
+    NoFileOpens = 53,
+    NoFileErrors = 54,
+    StallL0SlowdownMicros = 55,
+    StallMemtableCompactionMicros = 56,
+    StallL0NumFilesMicros = 57,
+    StallMicros = 58,
+    DbMutexWaitMicros = 59,
+    RateLimitDelayMillis = 60,
+    NoIterators = 61,
+    NumberMultigetCalls = 62,
+    NumberMultigetKeysRead = 63,
+    NumberMultigetBytesRead = 64,
+    NumberFilteredDeletes = 65,
+    NumberMergeFailures = 66,
+    BloomFilterPrefixChecked = 67,
+    BloomFilterPrefixUseful = 68,
+    NumberOfReseeksInIteration = 69,
+    GetUpdatesSinceCalls = 70,
+    BlockCacheCompressedMiss = 71,
+    BlockCacheCompressedHit = 72,
+    BlockCacheCompressedAdd = 73,
+    BlockCacheCompressedAddFailures = 74,
+    WalFileSynced = 75,
+    WalFileBytes = 76,
+    WriteDoneBySelf = 77,
+    WriteDoneByOther = 78,
+    WriteTimedout = 79,
+    WriteWithWal = 80,
+    CompactReadBytes = 81,
+    CompactWriteBytes = 82,
+    FlushWriteBytes = 83,
+    NumberDirectLoadTableProperties = 84,
+    NumberSuperversionAcquires = 85,
+    NumberSuperversionReleases = 86,
+    NumberSuperversionCleanups = 87,
+    NumberBlockCompressed = 88,
+    NumberBlockDecompressed = 89,
+    NumberBlockNotCompressed = 90,
+    MergeOperationTotalTime = 91,
+    FilterOperationTotalTime = 92,
+    RowCacheHit = 93,
+    RowCacheMiss = 94,
+    ReadAmpEstimateUsefulBytes = 95,
+    ReadAmpTotalReadBytes = 96,
+    NumberRateLimiterDrains = 97,
+    NumberIterSkip = 98,
+    BlobDbNumPut = 99,
+    BlobDbNumWrite = 100,
+    BlobDbNumGet = 101,
+    BlobDbNumMultiget = 102,
+    BlobDbNumSeek = 103,
+    BlobDbNumNext = 104,
+    BlobDbNumPrev = 105,
+    BlobDbNumKeysWritten = 106,
+    BlobDbNumKeysRead = 107,
+    BlobDbBytesWritten = 108,
+    BlobDbBytesRead = 109,
+    BlobDbWriteInlined = 110,
+    BlobDbWriteInlinedTtl = 111,
+    BlobDbWriteBlob = 112,
+    BlobDbWriteBlobTtl = 113,
+    BlobDbBlobFileBytesWritten = 114,
+    BlobDbBlobFileBytesRead = 115,
+    BlobDbBlobFileSynced = 116,
+    BlobDbBlobIndexExpiredCount = 117,
+    BlobDbBlobIndexExpiredSize = 118,
+    BlobDbBlobIndexEvictedCount = 119,
+    BlobDbBlobIndexEvictedSize = 120,
+    BlobDbGcNumFiles = 121,
+    BlobDbGcNumNewFiles = 122,
+    BlobDbGcFailures = 123,
+    BlobDbGcNumKeysOverwritten = 124,
+    BlobDbGcNumKeysExpired = 125,
+    BlobDbGcNumKeysRelocated = 126,
+    BlobDbGcBytesOverwritten = 127,
+    BlobDbGcBytesExpired = 128,
+    BlobDbGcBytesRelocated = 129,
+    BlobDbFifoNumFilesEvicted = 130,
+    BlobDbFifoNumKeysEvicted = 131,
+    BlobDbFifoBytesEvicted = 132,
+    TxnPrepareMutexOverhead = 133,
+    TxnOldCommitMapMutexOverhead = 134,
+    TxnDuplicateKeyOverhead = 135,
+    TxnSnapshotMutexOverhead = 136,
+    NumberMultigetKeysFound = 137,
+    NoIteratorCreated = 138,
+    NoIteratorDeleted = 139,
+    BlockCacheCompressionDictMiss = 140,
+    BlockCacheCompressionDictHit = 141,
+    BlockCacheCompressionDictAdd = 142,
+    BlockCacheCompressionDictBytesInsert = 143,
+    BlockCacheCompressionDictBytesEvict = 144,
 }
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[repr(C)]
@@ -148,48 +156,50 @@ pub enum DBStatisticsHistogramType {
     DbGet = 0,
     DbWrite = 1,
     CompactionTime = 2,
-    SubcompactionSetupTime = 3,
-    TableSyncMicros = 4,
-    CompactionOutfileSyncMicros = 5,
-    WalFileSyncMicros = 6,
-    ManifestFileSyncMicros = 7,
-    TableOpenIoMicros = 8,
-    DbMultiget = 9,
-    ReadBlockCompactionMicros = 10,
-    ReadBlockGetMicros = 11,
-    WriteRawBlockMicros = 12,
-    StallL0SlowdownCount = 13,
-    StallMemtableCompactionCount = 14,
-    StallL0NumFilesCount = 15,
-    HardRateLimitDelayCount = 16,
-    SoftRateLimitDelayCount = 17,
-    NumFilesInSingleCompaction = 18,
-    DbSeek = 19,
-    WriteStall = 20,
-    SstReadMicros = 21,
-    NumSubcompactionsScheduled = 22,
-    BytesPerRead = 23,
-    BytesPerWrite = 24,
-    BytesPerMultiget = 25,
-    BytesCompressed = 26,
-    BytesDecompressed = 27,
-    CompressionTimesNanos = 28,
-    DecompressionTimesNanos = 29,
-    ReadNumMergeOperands = 30,
-    BlobDbKeySize = 31,
-    BlobDbValueSize = 32,
-    BlobDbWriteMicros = 33,
-    BlobDbGetMicros = 34,
-    BlobDbMultigetMicros = 35,
-    BlobDbSeekMicros = 36,
-    BlobDbNextMicros = 37,
-    BlobDbPrevMicros = 38,
-    BlobDbBlobFileWriteMicros = 39,
-    BlobDbBlobFileReadMicros = 40,
-    BlobDbBlobFileSyncMicros = 41,
-    BlobDbGcMicros = 42,
-    BlobDbCompressionMicros = 43,
-    BlobDbDecompressionMicros = 44,
-    FlushTime = 45,
-    HistogramEnumMax = 46,
+    CompactionCpuTime = 3,
+    SubcompactionSetupTime = 4,
+    TableSyncMicros = 5,
+    CompactionOutfileSyncMicros = 6,
+    WalFileSyncMicros = 7,
+    ManifestFileSyncMicros = 8,
+    TableOpenIoMicros = 9,
+    DbMultiget = 10,
+    ReadBlockCompactionMicros = 11,
+    ReadBlockGetMicros = 12,
+    WriteRawBlockMicros = 13,
+    StallL0SlowdownCount = 14,
+    StallMemtableCompactionCount = 15,
+    StallL0NumFilesCount = 16,
+    HardRateLimitDelayCount = 17,
+    SoftRateLimitDelayCount = 18,
+    NumFilesInSingleCompaction = 19,
+    DbSeek = 20,
+    WriteStall = 21,
+    SstReadMicros = 22,
+    NumSubcompactionsScheduled = 23,
+    BytesPerRead = 24,
+    BytesPerWrite = 25,
+    BytesPerMultiget = 26,
+    BytesCompressed = 27,
+    BytesDecompressed = 28,
+    CompressionTimesNanos = 29,
+    DecompressionTimesNanos = 30,
+    ReadNumMergeOperands = 31,
+    BlobDbKeySize = 32,
+    BlobDbValueSize = 33,
+    BlobDbWriteMicros = 34,
+    BlobDbGetMicros = 35,
+    BlobDbMultigetMicros = 36,
+    BlobDbSeekMicros = 37,
+    BlobDbNextMicros = 38,
+    BlobDbPrevMicros = 39,
+    BlobDbBlobFileWriteMicros = 40,
+    BlobDbBlobFileReadMicros = 41,
+    BlobDbBlobFileSyncMicros = 42,
+    BlobDbGcMicros = 43,
+    BlobDbCompressionMicros = 44,
+    BlobDbDecompressionMicros = 45,
+    FlushTime = 46,
+    SstBatchSize = 47,
+    HistogramEnumMax = 48,
 }

librocksdb_sys/src/lib.rs

@@ -1146,10 +1146,6 @@ extern "C" {
         fifo_opts: *mut DBFifoCompactionOptions,
         size: uint64_t,
     );
-    pub fn crocksdb_fifo_compaction_options_set_ttl(
-        fifo_opts: *mut DBFifoCompactionOptions,
-        ttl: uint64_t,
-    );
     pub fn crocksdb_fifo_compaction_options_set_allow_compaction(
         fifo_opts: *mut DBFifoCompactionOptions,
         allow_compaction: bool,
@@ -1956,7 +1952,7 @@ extern "C" {
 
     pub fn ctitandb_options_set_disable_background_gc(opts: *mut DBTitanDBOptions, disable: bool);
     pub fn ctitandb_options_set_max_background_gc(opts: *mut DBTitanDBOptions, size: i32);
-    pub fn ctitandb_options_set_purge_obsolete_files_period(
+    pub fn ctitandb_options_set_purge_obsolete_files_period_sec(
         opts: *mut DBTitanDBOptions,
         period: usize,
     );

src/rocksdb_options.rs

@@ -1859,12 +1859,6 @@ impl FifoCompactionOptions {
         }
     }
 
-    pub fn set_ttl(&mut self, ttl: u64) {
-        unsafe {
-            crocksdb_ffi::crocksdb_fifo_compaction_options_set_ttl(self.inner, ttl);
-        }
-    }
-
     pub fn set_max_table_files_size(&mut self, max_table_files_size: u64) {
         unsafe {
             crocksdb_ffi::crocksdb_fifo_compaction_options_set_max_table_files_size(

src/titan.rs

@@ -72,7 +72,7 @@ impl TitanDBOptions {
 
     pub fn set_purge_obsolete_files_period(&mut self, period: usize) {
         unsafe {
-            crocksdb_ffi::ctitandb_options_set_purge_obsolete_files_period(self.inner, period);
+            crocksdb_ffi::ctitandb_options_set_purge_obsolete_files_period_sec(self.inner, period);
         }
     }
 

tests/cases/test_rocksdb_options.rs

@@ -772,7 +772,6 @@ fn test_fifo_compaction_options() {
     let mut cf_opts = ColumnFamilyOptions::new();
     let mut fifo_opts = FifoCompactionOptions::new();
     fifo_opts.set_allow_compaction(true);
-    fifo_opts.set_ttl(100000);
     fifo_opts.set_max_table_files_size(100000);
     cf_opts.set_fifo_compaction_options(fifo_opts);
     DB::open_cf(opts, path_str, vec![("default", cf_opts)]).unwrap();