*: generate some enums with a script (#219)

New version rocksdb may change the value of some enums, then we have
to update those enums accordingly. So we need a script to generate
those enums instead of updating them manually.

librocksdb_sys/src/generate.py

+#!/usr/bin/env python3
+
+import re
+import sys
+
+root = '../rocksdb/include/rocksdb'
+
+tasks = [
+    (
+        'DBStatisticsTickerType',
+        'statistics.h',
+        re.compile(r'enum Tickers .* {'),
+        re.compile(r'};\s*'),
+        re.compile(r'\s*\w(_\w)*.*,'),
+    ),
+    (
+        'DBStatisticsHistogramType',
+        'statistics.h',
+        re.compile(r'enum Histograms .* {'),
+        re.compile(r'};\s*'),
+        re.compile(r'\s*\w(_\w)*.*,'),
+    ),
+]
+
+print('/// This file is generated from generate.py.')
+print('/// Re-generate it if you upgrade to a new version of RocksDB.')
+print('')
+
+for task in tasks:
+    begin = False
+    count = 0
+    for line in open(root + '/' + task[1]):
+        if not begin:
+            if task[2].match(line):
+                begin = True
+                print('#[derive(Copy, Clone, Debug, Eq, PartialEq)]')
+                print('#[repr(C)]')
+                print('pub enum {} {{'.format(task[0]))
+            continue
+        if task[3].match(line):
+            print('}')
+            break
+        if not task[4].match(line):
+            continue
+        tokens = line.split(',')[0].split('=')
+        if len(tokens) == 1:
+            name = tokens[0].strip(' ')
+            value = count
+        elif len(tokens) == 2:
+            name = tokens[0].strip(' ')
+            value = tokens[1].strip(' ')
+            count = int(value)
+        else:
+            sys.exit("invalid enum: " + line)
+        name = ''.join([w.capitalize() for w in name.split('_')])
+        count = count + 1
+        print('    {} = {},'.format(name, value))

librocksdb_sys/src/generated.rs

+/// This file is generated from generate.py.
+/// Re-generate it if you upgrade to a new version of RocksDB.
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+#[repr(C)]
+pub enum DBStatisticsTickerType {
+    BlockCacheMiss = 0,
+    BlockCacheHit = 1,
+    BlockCacheAdd = 2,
+    BlockCacheAddFailures = 3,
+    BlockCacheIndexMiss = 4,
+    BlockCacheIndexHit = 5,
+    BlockCacheIndexAdd = 6,
+    BlockCacheIndexBytesInsert = 7,
+    BlockCacheIndexBytesEvict = 8,
+    BlockCacheFilterMiss = 9,
+    BlockCacheFilterHit = 10,
+    BlockCacheFilterAdd = 11,
+    BlockCacheFilterBytesInsert = 12,
+    BlockCacheFilterBytesEvict = 13,
+    BlockCacheDataMiss = 14,
+    BlockCacheDataHit = 15,
+    BlockCacheDataAdd = 16,
+    BlockCacheDataBytesInsert = 17,
+    BlockCacheBytesRead = 18,
+    BlockCacheBytesWrite = 19,
+    BloomFilterUseful = 20,
+    PersistentCacheHit = 21,
+    PersistentCacheMiss = 22,
+    SimBlockCacheHit = 23,
+    SimBlockCacheMiss = 24,
+    MemtableHit = 25,
+    MemtableMiss = 26,
+    GetHitL0 = 27,
+    GetHitL1 = 28,
+    GetHitL2AndUp = 29,
+    CompactionKeyDropNewerEntry = 30,
+    CompactionKeyDropObsolete = 31,
+    CompactionKeyDropRangeDel = 32,
+    CompactionKeyDropUser = 33,
+    CompactionRangeDelDropObsolete = 34,
+    CompactionOptimizedDelDropObsolete = 35,
+    NumberKeysWritten = 36,
+    NumberKeysRead = 37,
+    NumberKeysUpdated = 38,
+    BytesWritten = 39,
+    BytesRead = 40,
+    NumberDbSeek = 41,
+    NumberDbNext = 42,
+    NumberDbPrev = 43,
+    NumberDbSeekFound = 44,
+    NumberDbNextFound = 45,
+    NumberDbPrevFound = 46,
+    IterBytesRead = 47,
+    NoFileCloses = 48,
+    NoFileOpens = 49,
+    NoFileErrors = 50,
+    StallL0SlowdownMicros = 51,
+    StallMemtableCompactionMicros = 52,
+    StallL0NumFilesMicros = 53,
+    StallMicros = 54,
+    DbMutexWaitMicros = 55,
+    RateLimitDelayMillis = 56,
+    NoIterators = 57,
+    NumberMultigetCalls = 58,
+    NumberMultigetKeysRead = 59,
+    NumberMultigetBytesRead = 60,
+    NumberFilteredDeletes = 61,
+    NumberMergeFailures = 62,
+    BloomFilterPrefixChecked = 63,
+    BloomFilterPrefixUseful = 64,
+    NumberOfReseeksInIteration = 65,
+    GetUpdatesSinceCalls = 66,
+    BlockCacheCompressedMiss = 67,
+    BlockCacheCompressedHit = 68,
+    BlockCacheCompressedAdd = 69,
+    BlockCacheCompressedAddFailures = 70,
+    WalFileSynced = 71,
+    WalFileBytes = 72,
+    WriteDoneBySelf = 73,
+    WriteDoneByOther = 74,
+    WriteTimedout = 75,
+    WriteWithWal = 76,
+    CompactReadBytes = 77,
+    CompactWriteBytes = 78,
+    FlushWriteBytes = 79,
+    NumberDirectLoadTableProperties = 80,
+    NumberSuperversionAcquires = 81,
+    NumberSuperversionReleases = 82,
+    NumberSuperversionCleanups = 83,
+    NumberBlockCompressed = 84,
+    NumberBlockDecompressed = 85,
+    NumberBlockNotCompressed = 86,
+    MergeOperationTotalTime = 87,
+    FilterOperationTotalTime = 88,
+    RowCacheHit = 89,
+    RowCacheMiss = 90,
+    ReadAmpEstimateUsefulBytes = 91,
+    ReadAmpTotalReadBytes = 92,
+    NumberRateLimiterDrains = 93,
+}
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+#[repr(C)]
+pub enum DBStatisticsHistogramType {
+    DbGet = 0,
+    DbWrite = 1,
+    CompactionTime = 2,
+    SubcompactionSetupTime = 3,
+    TableSyncMicros = 4,
+    CompactionOutfileSyncMicros = 5,
+    WalFileSyncMicros = 6,
+    ManifestFileSyncMicros = 7,
+    TableOpenIoMicros = 8,
+    DbMultiget = 9,
+    ReadBlockCompactionMicros = 10,
+    ReadBlockGetMicros = 11,
+    WriteRawBlockMicros = 12,
+    StallL0SlowdownCount = 13,
+    StallMemtableCompactionCount = 14,
+    StallL0NumFilesCount = 15,
+    HardRateLimitDelayCount = 16,
+    SoftRateLimitDelayCount = 17,
+    NumFilesInSingleCompaction = 18,
+    DbSeek = 19,
+    WriteStall = 20,
+    SstReadMicros = 21,
+    NumSubcompactionsScheduled = 22,
+    BytesPerRead = 23,
+    BytesPerWrite = 24,
+    BytesPerMultiget = 25,
+    BytesCompressed = 26,
+    BytesDecompressed = 27,
+    CompressionTimesNanos = 28,
+    DecompressionTimesNanos = 29,
+    ReadNumMergeOperands = 30,
+    HistogramEnumMax = 31,
+}

librocksdb_sys/src/lib.rs

@@ -69,6 +69,9 @@ pub enum DBSstFileMetaData {}
 pub enum DBCompactionOptions {}
 pub enum DBPerfContext {}
 
+mod generated;
+pub use generated::*;
+
 pub fn new_bloom_filter(bits: c_int) -> *mut DBFilterPolicy {
     unsafe { crocksdb_filterpolicy_create_bloom(bits) }
 }
@@ -152,113 +155,6 @@ pub enum CompactionPriority {
     MinOverlappingRatio = 3,
 }
 
-#[derive(Copy, Clone, Debug, Eq, PartialEq)]
-#[repr(C)]
-pub enum DBStatisticsTickerType {
-    BlockCacheMiss = 0, // total block cache miss
-    BlockCacheHit = 1,  // total block cache hit
-    BlockCacheAdd = 2,
-    BlockCacheAddFailures = 3,
-    BlockCacheIndexMiss = 4, // times cache miss when accessing index block from block cache
-    BlockCacheIndexHit = 5,
-    BlockCacheIndexAdd = 6,
-    BlockCacheIndexBytesInsert = 7,
-    BlockCacheIndexBytesEvict = 8,
-    BlockCacheFilterMiss = 9, // times cache miss when accessing filter block from block cache
-    BlockCacheFilterHit = 10,
-    BlockCacheFilterAdd = 11,
-    BlockCacheFilterBytesInsert = 12,
-    BlockCacheFilterBytesEvict = 13,
-    BlockCacheDataMiss = 14, // times cache miss when accessing data block from block cache
-    BlockCacheDataHit = 15,  // times cache hit when accessing data block from block cache
-    BlockCacheDataAdd = 16,
-    BlockCacheDataBytesInsert = 17,
-    BlockCacheByteRead = 18,  // bytes read from cache
-    BlockCacheByteWrite = 19, // bytes written into cache
-    BloomFilterUseful = 20,   // times bloom filter has avoided file reads
-    MemtableHit = 25,
-    MemtableMiss = 26,
-    GetHitL0 = 27,      // Get() queries served by L0
-    GetHitL1 = 28,      // Get() queries served by L1
-    GetHitL2AndUp = 29, // Get() queries served by L2 and up
-    CompactionKeyDropNewerEntry = 30, /* key was written with a newer value.
-                         * Also includes keys dropped for range del. */
-    CompactionKeyDropObsolete = 31,          // The key is obsolete.
-    CompactionKeyDropRangeDel = 32,          // key was covered by a range tombstone.
-    CompactionRangeDelDropObsolete = 34,     // all keys in range were deleted.
-    CompactionOptimizedDelDropObsolete = 35, // Deletions obsoleted before bottom level due to file gap optimization.
-    NumberKeysWritten = 36, // number of keys written to the database via the Put and Write call's
-    NumberKeysRead = 37,    // number of keys read
-    NumberKeysUpdated = 38,
-    BytesWritten = 39, // the number of uncompressed bytes read from DB::Put, DB::Delete,
-    // DB::Merge and DB::Write
-    BytesRead = 40,    // the number of uncompressed bytes read from DB::Get()
-    NumberDbSeek = 41, // the number of calls to seek/next/prev
-    NumberDbNext = 42,
-    NumberDbPrev = 43,
-    NumberDbSeekFound = 44, // the number of calls to seek/next/prev that returned data
-    NumberDbNextFound = 45,
-    NumberDbPrevFound = 46,
-    IterBytesRead = 47, // the number of uncompressed bytes read from an iterator, include size of
-    // key and value
-    NoFileCloses = 48,
-    NoFileOpens = 49,
-    NoFileErrors = 50,
-    StallMicros = 54, // writer has to wait for compaction or flush to finish
-    NoIterators = 57, // number of iterators currently open
-    BloomFilterPrefixChecked = 63, // number of times bloom was checked before creating iterator
-    // on a file
-    BloomFilterPrefixUseful = 64, // number of times the check was useful in avoiding iterator
-    // creating
-    WalFileSynced = 71,              // number of times WAL sync is done
-    WalFileBytes = 72,               // number of bytes written to WAL
-    WriteDoneBySelf = 73,            // number of writes processed by self
-    WriteDoneByOther = 74,           // number of writes processed by other
-    WriteTimeout = 75,               // number of writes ending up with timed-out
-    WriteWithWAL = 76,               // number of writes that request WAL
-    CompactReadBytes = 77,           // bytes read during compaction
-    CompactWriteBytes = 78,          // bytes written during compaction
-    FlushWriteBytes = 79,            // bytes written during flush
-    ReadAmpEstimateUsefulBytes = 91, // total bytes actually used
-    ReadAmpTotalReadBytes = 92,      // total size of loaded data blocks
-}
-
-#[derive(Copy, Clone, Debug, Eq, PartialEq)]
-#[repr(C)]
-pub enum DBStatisticsHistogramType {
-    GetMicros = 0,
-    WriteMicros = 1,
-    CompactionTime = 2,
-    SubcompactionSetupTime = 3,
-    TableSyncMicros = 4,
-    CompactionOutfileSyncMicros = 5,
-    WalFileSyncMicros = 6,
-    ManifestFileSyncMicros = 7,
-    TableOpenIOMicros = 8,
-    MultiGetMicros = 9,
-    ReadBlockCompactionMicros = 10,
-    ReadBlockGetMicros = 11,
-    WriteRawBlockMicros = 12,
-    StallL0SlowdownCount = 13,
-    StallMemtableCompactionCount = 14,
-    StallL0NumFilesCount = 15,
-    HardRateLimitDelayCount = 16,
-    SoftRateLimitDelayCount = 17,
-    NumFilesInSingleCompaction = 18,
-    SeekMicros = 19,
-    WriteStall = 20,
-    SSTReadMicros = 21,
-    NumSubcompactionsScheduled = 22,
-    BytesPerRead = 23,
-    BytesPerWrite = 24,
-    BytesPerMultiget = 25,
-    BytesCompressed = 26,
-    BytesDecompressed = 27,
-    CompressionTimesNanos = 28,
-    DecompressionTimesNanos = 29,
-    ReadNumMergeOperands = 30,
-}
-
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[repr(C)]
 pub enum DBInfoLogLevel {

tests/test_rocksdb_options.rs

@@ -75,11 +75,11 @@ fn test_enable_statistics() {
     opts.set_stats_dump_period_sec(60);
     assert!(opts.get_statistics().is_some());
     assert!(
-        opts.get_statistics_histogram(HistogramType::SeekMicros)
+        opts.get_statistics_histogram(HistogramType::DbSeek)
             .is_some()
     );
     assert!(
-        opts.get_statistics_histogram_string(HistogramType::SeekMicros)
+        opts.get_statistics_histogram_string(HistogramType::DbSeek)
             .is_some()
     );
     assert_eq!(

tests/test_statistics.rs

@@ -36,22 +36,15 @@ fn test_db_statistics() {
     assert!(db.get_and_reset_statistics_ticker_count(TickerType::BlockCacheHit) > 0);
     assert_eq!(db.get_statistics_ticker_count(TickerType::BlockCacheHit), 0);
     assert!(
-        db.get_statistics_histogram_string(HistogramType::GetMicros)
-            .is_some()
-    );
-    assert!(
-        db.get_statistics_histogram(HistogramType::GetMicros)
+        db.get_statistics_histogram_string(HistogramType::DbGet)
             .is_some()
     );
+    assert!(db.get_statistics_histogram(HistogramType::DbGet).is_some());
 
-    let get_micros = db
-        .get_statistics_histogram(HistogramType::GetMicros)
-        .unwrap();
+    let get_micros = db.get_statistics_histogram(HistogramType::DbGet).unwrap();
     assert!(get_micros.max > 0.0);
     db.reset_statistics();
-    let get_micros = db
-        .get_statistics_histogram(HistogramType::GetMicros)
-        .unwrap();
+    let get_micros = db.get_statistics_histogram(HistogramType::DbGet).unwrap();
     assert_eq!(get_micros.max, 0.0);
 }
 
@@ -78,11 +71,8 @@ fn test_disable_db_statistics() {
         0
     );
     assert!(
-        db.get_statistics_histogram_string(HistogramType::GetMicros)
-            .is_none()
-    );
-    assert!(
-        db.get_statistics_histogram(HistogramType::GetMicros)
+        db.get_statistics_histogram_string(HistogramType::DbGet)
             .is_none()
     );
+    assert!(db.get_statistics_histogram(HistogramType::DbGet).is_none());
 }