Add more metrics (#79)

* add more metrics
Signed-off-by: Connor1996 <zbk602423539@gmail.com>

include/titan/db.h

@@ -158,6 +158,21 @@ class TitanDB : public StackableDB {
     //  "rocksdb.titandb.obsolete-blob-file-size" - returns size of obsolete
     //      blob files.
     static const std::string kObsoleteBlobFileSize;
+    //  "rocksdb.titandb.discardable_ratio_le0_file_num" - returns count of
+    //  file whose discardable ratio is less or equal to 0%.
+    static const std::string kNumDiscardableRatioLE0File;
+    //  "rocksdb.titandb.discardable_ratio_le20_file_num" - returns count of
+    //  file whose discardable ratio is less or equal to 20%.
+    static const std::string kNumDiscardableRatioLE20File;
+    //  "rocksdb.titandb.discardable_ratio_le50_file_num" - returns count of
+    //  file whose discardable ratio is less or equal to 50%.
+    static const std::string kNumDiscardableRatioLE50File;
+    //  "rocksdb.titandb.discardable_ratio_le80_file_num" - returns count of
+    //  file whose discardable ratio is less or equal to 80%.
+    static const std::string kNumDiscardableRatioLE80File;
+    //  "rocksdb.titandb.discardable_ratio_le100_file_num" - returns count of
+    //  file whose discardable ratio is less or equal to 100%.
+    static const std::string kNumDiscardableRatioLE100File;
   };
 
   bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property,

src/blob_file_reader.cc

@@ -106,15 +106,13 @@ Status BlobFileReader::Get(const ReadOptions& /*options*/,
     EncodeBlobCache(&cache_key, cache_prefix_, handle.offset);
     cache_handle = cache_->Lookup(cache_key);
     if (cache_handle) {
-      RecordTick(stats_, BLOCK_CACHE_DATA_HIT);
-      RecordTick(stats_, BLOCK_CACHE_HIT);
+      RecordTick(stats_, TitanStats::BLOB_CACHE_HIT);
       auto blob = reinterpret_cast<OwnedSlice*>(cache_->Value(cache_handle));
       buffer->PinSlice(*blob, UnrefCacheHandle, cache_.get(), cache_handle);
       return DecodeInto(*blob, record);
     }
   }
-  RecordTick(stats_, BLOCK_CACHE_DATA_MISS);
-  RecordTick(stats_, BLOCK_CACHE_MISS);
+  RecordTick(stats_, TitanStats::BLOB_CACHE_MISS);
 
   OwnedSlice blob;
   Status s = ReadRecord(handle, record, &blob);

src/blob_file_set.cc

@@ -2,8 +2,6 @@
 
 #include <inttypes.h>
 
-#include "file/filename.h"
-
 #include "edit_collector.h"
 
 namespace rocksdb {
@@ -163,7 +161,7 @@ Status BlobFileSet::OpenManifest(uint64_t file_number) {
   s = WriteSnapshot(manifest_.get());
   if (s.ok()) {
     ImmutableDBOptions ioptions(db_options_);
-    s = SyncManifest(env_, &ioptions, manifest_->file());
+    s = SyncTitanManifest(env_, stats_, &ioptions, manifest_->file());
   }
   if (s.ok()) {
     // Makes "CURRENT" file that points to the new manifest file.
@@ -223,7 +221,7 @@ Status BlobFileSet::LogAndApply(VersionEdit& edit) {
   if (!s.ok()) return s;
 
   ImmutableDBOptions ioptions(db_options_);
-  s = SyncManifest(env_, &ioptions, manifest_->file());
+  s = SyncTitanManifest(env_, stats_, &ioptions, manifest_->file());
   if (!s.ok()) return s;
   return collector.Apply(*this);
 }

src/blob_format.cc

@@ -229,6 +229,27 @@ void BlobFileMeta::AddDiscardableSize(uint64_t _discardable_size) {
   assert(discardable_size_ < file_size_);
 }
 
+TitanInternalStats::StatsType BlobFileMeta::GetDiscardableRatioLevel() const {
+  auto ratio = GetDiscardableRatio();
+  TitanInternalStats::StatsType type;
+  if (ratio == 0) {
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE0;
+  } else if (ratio <= 0.2) {
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE20;
+  } else if (ratio <= 0.5) {
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE50;
+  } else if (ratio <= 0.8) {
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE80;
+  } else if (ratio <= 1.0 ||
+             (ratio - 1.0) < std::numeric_limits<double>::epsilon()) {
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100;
+  } else {
+    fprintf(stderr, "invalid discarable ratio");
+    abort();
+  }
+  return type;
+}
+
 double BlobFileMeta::GetDiscardableRatio() const {
   return static_cast<double>(discardable_size_) /
          static_cast<double>(file_size_);

src/blob_format.h

@@ -217,6 +217,7 @@ class BlobFileMeta {
 
   void AddDiscardableSize(uint64_t _discardable_size);
   double GetDiscardableRatio() const;
+  TitanInternalStats::StatsType GetDiscardableRatioLevel() const;
 
  private:
   // Persistent field

src/blob_gc_job.cc

@@ -97,19 +97,20 @@ BlobGCJob::~BlobGCJob() {
     LogFlush(db_options_.info_log.get());
   }
   // flush metrics
-  RecordTick(stats_, BLOB_DB_BYTES_READ, metrics_.blob_db_bytes_read);
-  RecordTick(stats_, BLOB_DB_BYTES_WRITTEN, metrics_.blob_db_bytes_written);
+  RecordTick(stats_, BLOB_DB_BYTES_READ, metrics_.bytes_read);
+  RecordTick(stats_, BLOB_DB_BYTES_WRITTEN, metrics_.bytes_written);
   RecordTick(stats_, BLOB_DB_GC_NUM_KEYS_OVERWRITTEN,
-             metrics_.blob_db_gc_num_keys_overwritten);
+             metrics_.gc_num_keys_overwritten);
   RecordTick(stats_, BLOB_DB_GC_BYTES_OVERWRITTEN,
-             metrics_.blob_db_gc_bytes_overwritten);
+             metrics_.gc_bytes_overwritten);
   RecordTick(stats_, BLOB_DB_GC_NUM_KEYS_RELOCATED,
-             metrics_.blob_db_gc_num_keys_relocated);
-  RecordTick(stats_, BLOB_DB_GC_BYTES_RELOCATED,
-             metrics_.blob_db_gc_bytes_relocated);
-  RecordTick(stats_, BLOB_DB_GC_NUM_NEW_FILES,
-             metrics_.blob_db_gc_num_new_files);
-  RecordTick(stats_, BLOB_DB_GC_NUM_FILES, metrics_.blob_db_gc_num_files);
+             metrics_.gc_num_keys_relocated);
+  RecordTick(stats_, BLOB_DB_GC_BYTES_RELOCATED, metrics_.gc_bytes_relocated);
+  RecordTick(stats_, BLOB_DB_GC_NUM_NEW_FILES, metrics_.gc_num_new_files);
+  RecordTick(stats_, BLOB_DB_GC_NUM_FILES, metrics_.gc_num_files);
+  RecordTick(stats_, TitanStats::GC_DISCARDABLE, metrics_.gc_discardable);
+  RecordTick(stats_, TitanStats::GC_SMALL_FILE, metrics_.gc_small_file);
+  RecordTick(stats_, TitanStats::GC_SAMPLE, metrics_.gc_sample);
 }
 
 Status BlobGCJob::Prepare() {
@@ -151,7 +152,7 @@ Status BlobGCJob::Run() {
 }
 
 Status BlobGCJob::SampleCandidateFiles() {
-  TitanStopWatch sw(env_, metrics_.blob_db_gc_sampling_micros);
+  TitanStopWatch sw(env_, metrics_.gc_sampling_micros);
   std::vector<BlobFileMeta*> result;
   for (const auto& file : blob_gc_->inputs()) {
     bool selected = false;
@@ -172,12 +173,15 @@ Status BlobGCJob::SampleCandidateFiles() {
 Status BlobGCJob::DoSample(const BlobFileMeta* file, bool* selected) {
   assert(selected != nullptr);
   if (file->file_size() <=
-          blob_gc_->titan_cf_options().merge_small_file_threshold ||
-      file->GetDiscardableRatio() >=
-          blob_gc_->titan_cf_options().blob_file_discardable_ratio) {
+      blob_gc_->titan_cf_options().merge_small_file_threshold) {
+    metrics_.gc_small_file += 1;
+    *selected = true;
+  } else if (file->GetDiscardableRatio() >=
+             blob_gc_->titan_cf_options().blob_file_discardable_ratio) {
+    metrics_.gc_discardable += 1;
     *selected = true;
-    return Status::OK();
   }
+  if (*selected) return Status::OK();
 
   // TODO: add do sample count metrics
   auto records_size = file->file_size() - BlobFileHeader::kEncodedLength -
@@ -232,7 +236,7 @@ Status BlobGCJob::DoSample(const BlobFileMeta* file, bool* selected) {
       discardable_size += total_length;
     }
   }
-  metrics_.blob_db_bytes_read += iterated_size;
+  metrics_.bytes_read += iterated_size;
   assert(iter.status().ok());
 
   *selected =
@@ -281,7 +285,7 @@ Status BlobGCJob::DoRunGC() {
     }
     BlobIndex blob_index = gc_iter->GetBlobIndex();
     // count read bytes for blob record of gc candidate files
-    metrics_.blob_db_bytes_read += blob_index.blob_handle.size;
+    metrics_.bytes_read += blob_index.blob_handle.size;
 
     if (!last_key.empty() && !gc_iter->key().compare(last_key)) {
       if (last_key_valid) {
@@ -298,8 +302,8 @@ Status BlobGCJob::DoRunGC() {
       break;
     }
     if (discardable) {
-      metrics_.blob_db_gc_num_keys_overwritten++;
-      metrics_.blob_db_gc_bytes_overwritten += blob_index.blob_handle.size;
+      metrics_.gc_num_keys_overwritten++;
+      metrics_.gc_bytes_overwritten += blob_index.blob_handle.size;
       continue;
     }
 
@@ -335,7 +339,7 @@ Status BlobGCJob::DoRunGC() {
     blob_record.value = gc_iter->value();
     // count written bytes for new blob record,
     // blob index's size is counted in `RewriteValidKeyToLSM`
-    metrics_.blob_db_bytes_written += blob_record.size();
+    metrics_.bytes_written += blob_record.size();
 
     BlobIndex new_blob_index;
     new_blob_index.file_number = blob_file_handle->GetNumber();
@@ -401,7 +405,7 @@ Status BlobGCJob::BuildIterator(
 
 Status BlobGCJob::DiscardEntry(const Slice& key, const BlobIndex& blob_index,
                                bool* discardable) {
-  TitanStopWatch sw(env_, metrics_.blob_db_gc_read_lsm_micros);
+  TitanStopWatch sw(env_, metrics_.gc_read_lsm_micros);
   assert(discardable != nullptr);
   PinnableSlice index_entry;
   bool is_blob_index = false;
@@ -412,7 +416,7 @@ Status BlobGCJob::DiscardEntry(const Slice& key, const BlobIndex& blob_index,
     return s;
   }
   // count read bytes for checking LSM entry
-  metrics_.blob_db_bytes_read += key.size() + index_entry.size();
+  metrics_.bytes_read += key.size() + index_entry.size();
   if (s.IsNotFound() || !is_blob_index) {
     // Either the key is deleted or updated with a newer version which is
     // inlined in LSM.
@@ -475,7 +479,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
     if (!s.ok()) {
       break;
     }
-    metrics_.blob_db_gc_num_new_files++;
+    metrics_.gc_num_new_files++;
   }
   if (s.ok()) {
     std::vector<std::pair<std::shared_ptr<BlobFileMeta>,
@@ -487,7 +491,8 @@ Status BlobGCJob::InstallOutputBlobFiles() {
           builder.first->GetNumber(), builder.first->GetFile()->GetFileSize(),
           0, 0, builder.second->GetSmallestKey(),
           builder.second->GetLargestKey());
-
+      RecordInHistogram(stats_, TitanStats::GC_OUTPUT_FILE_SIZE,
+                        file->file_size());
       if (!tmp.empty()) {
         tmp.append(" ");
       }
@@ -525,7 +530,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
 }
 
 Status BlobGCJob::RewriteValidKeyToLSM() {
-  TitanStopWatch sw(env_, metrics_.blob_db_gc_update_lsm_micros);
+  TitanStopWatch sw(env_, metrics_.gc_update_lsm_micros);
   Status s;
   auto* db_impl = reinterpret_cast<DBImpl*>(this->base_db_);
 
@@ -544,22 +549,20 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
     s = db_impl->WriteWithCallback(wo, &write_batch.first, &write_batch.second);
     if (s.ok()) {
       // count written bytes for new blob index.
-      metrics_.blob_db_bytes_written += write_batch.first.GetDataSize();
-      metrics_.blob_db_gc_num_keys_relocated++;
-      metrics_.blob_db_gc_bytes_relocated +=
-          write_batch.second.blob_record_size();
+      metrics_.bytes_written += write_batch.first.GetDataSize();
+      metrics_.gc_num_keys_relocated++;
+      metrics_.gc_bytes_relocated += write_batch.second.blob_record_size();
       // Key is successfully written to LSM.
     } else if (s.IsBusy()) {
-      metrics_.blob_db_gc_num_keys_overwritten++;
-      metrics_.blob_db_gc_bytes_overwritten +=
-          write_batch.second.blob_record_size();
+      metrics_.gc_num_keys_overwritten++;
+      metrics_.gc_bytes_overwritten += write_batch.second.blob_record_size();
       // The key is overwritten in the meanwhile. Drop the blob record.
     } else {
       // We hit an error.
       break;
     }
     // count read bytes in write callback
-    metrics_.blob_db_bytes_read += write_batch.second.read_bytes();
+    metrics_.bytes_read += write_batch.second.read_bytes();
   }
   if (s.IsBusy()) {
     s = Status::OK();
@@ -583,7 +586,9 @@ Status BlobGCJob::DeleteInputBlobFiles() {
     ROCKS_LOG_INFO(db_options_.info_log,
                    "Titan add obsolete file [%" PRIu64 "]",
                    file->file_number());
-    metrics_.blob_db_gc_num_files++;
+    metrics_.gc_num_files++;
+    RecordInHistogram(stats_, TitanStats::GC_INPUT_FILE_SIZE,
+                      file->file_size());
     edit.DeleteBlobFile(file->file_number(), obsolete_sequence);
   }
   s = blob_file_set_->LogAndApply(edit);
@@ -612,23 +617,23 @@ void BlobGCJob::UpdateInternalOpStats() {
   assert(internal_op_stats != nullptr);
   AddStats(internal_op_stats, InternalOpStatsType::COUNT);
   AddStats(internal_op_stats, InternalOpStatsType::BYTES_READ,
-           metrics_.blob_db_bytes_read);
+           metrics_.bytes_read);
   AddStats(internal_op_stats, InternalOpStatsType::BYTES_WRITTEN,
-           metrics_.blob_db_bytes_written);
+           metrics_.bytes_written);
   AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_READ,
            io_bytes_read_);
   AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_WRITTEN,
            io_bytes_written_);
   AddStats(internal_op_stats, InternalOpStatsType::INPUT_FILE_NUM,
-           metrics_.blob_db_gc_num_files);
+           metrics_.gc_num_files);
   AddStats(internal_op_stats, InternalOpStatsType::OUTPUT_FILE_NUM,
-           metrics_.blob_db_gc_num_new_files);
+           metrics_.gc_num_new_files);
   AddStats(internal_op_stats, InternalOpStatsType::GC_SAMPLING_MICROS,
-           metrics_.blob_db_gc_sampling_micros);
+           metrics_.gc_sampling_micros);
   AddStats(internal_op_stats, InternalOpStatsType::GC_READ_LSM_MICROS,
-           metrics_.blob_db_gc_read_lsm_micros);
+           metrics_.gc_read_lsm_micros);
   AddStats(internal_op_stats, InternalOpStatsType::GC_UPDATE_LSM_MICROS,
-           metrics_.blob_db_gc_update_lsm_micros);
+           metrics_.gc_update_lsm_micros);
 }
 
 }  // namespace titandb

src/blob_gc_job.h

@@ -64,17 +64,20 @@ class BlobGCJob {
   TitanStats* stats_;
 
   struct {
-    uint64_t blob_db_bytes_read = 0;
-    uint64_t blob_db_bytes_written = 0;
-    uint64_t blob_db_gc_num_keys_overwritten = 0;
-    uint64_t blob_db_gc_bytes_overwritten = 0;
-    uint64_t blob_db_gc_num_keys_relocated = 0;
-    uint64_t blob_db_gc_bytes_relocated = 0;
-    uint64_t blob_db_gc_num_new_files = 0;
-    uint64_t blob_db_gc_num_files = 0;
-    uint64_t blob_db_gc_sampling_micros = 0;
-    uint64_t blob_db_gc_read_lsm_micros = 0;
-    uint64_t blob_db_gc_update_lsm_micros = 0;
+    uint64_t bytes_read = 0;
+    uint64_t bytes_written = 0;
+    uint64_t gc_num_keys_overwritten = 0;
+    uint64_t gc_bytes_overwritten = 0;
+    uint64_t gc_num_keys_relocated = 0;
+    uint64_t gc_bytes_relocated = 0;
+    uint64_t gc_num_new_files = 0;
+    uint64_t gc_num_files = 0;
+    uint64_t gc_small_file = 0;
+    uint64_t gc_discardable = 0;
+    uint64_t gc_sample = 0;
+    uint64_t gc_sampling_micros = 0;
+    uint64_t gc_read_lsm_micros = 0;
+    uint64_t gc_update_lsm_micros = 0;
   } metrics_;
 
   uint64_t prev_bytes_read_ = 0;

src/blob_gc_job_test.cc

@@ -118,7 +118,7 @@ class BlobGCJobTest : public testing::Test {
     std::unique_ptr<BlobGC> blob_gc;
     {
       std::shared_ptr<BlobGCPicker> blob_gc_picker =
-          std::make_shared<BasicBlobGCPicker>(db_options, cf_options);
+          std::make_shared<BasicBlobGCPicker>(db_options, cf_options, nullptr);
       blob_gc = blob_gc_picker->PickBlobGC(
           blob_file_set_->GetBlobStorage(cfh->GetID()).lock().get());
     }

src/blob_gc_picker.cc

@@ -10,8 +10,9 @@ namespace rocksdb {
 namespace titandb {
 
 BasicBlobGCPicker::BasicBlobGCPicker(TitanDBOptions db_options,
-                                     TitanCFOptions cf_options)
-    : db_options_(db_options), cf_options_(cf_options) {}
+                                     TitanCFOptions cf_options,
+                                     TitanStats* stats)
+    : db_options_(db_options), cf_options_(cf_options), stats_(stats) {}
 
 BasicBlobGCPicker::~BasicBlobGCPicker() {}
 
@@ -27,14 +28,11 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
   uint64_t next_gc_size = 0;
   for (auto& gc_score : blob_storage->gc_score()) {
     auto blob_file = blob_storage->FindFile(gc_score.file_number).lock();
-    if (!blob_file ||
-        blob_file->file_state() == BlobFileMeta::FileState::kBeingGC) {
+    if (!CheckBlobFile(blob_file.get())) {
+      RecordTick(stats_, TitanStats::GC_NO_NEED, 1);
       // Skip this file id this file is being GCed
       // or this file had been GCed
-      continue;
-    }
-    if (!CheckBlobFile(blob_file.get())) {
-      ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 "  no need gc",
+      ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 " no need gc",
                      blob_file->file_number());
       continue;
     }
@@ -57,6 +55,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
         next_gc_size += blob_file->file_size();
         if (next_gc_size > cf_options_.min_gc_batch_size) {
           maybe_continue_next_time = true;
+          RecordTick(stats_, TitanStats::GC_REMAIN, 1);
           ROCKS_LOG_INFO(db_options_.info_log,
                          "remain more than %" PRIu64
                          " bytes to be gc and trigger after this gc",
@@ -80,8 +79,11 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
 }
 
 bool BasicBlobGCPicker::CheckBlobFile(BlobFileMeta* blob_file) const {
-  assert(blob_file->file_state() != BlobFileMeta::FileState::kInit);
-  if (blob_file->file_state() != BlobFileMeta::FileState::kNormal) return false;
+  assert(blob_file != nullptr &&
+         blob_file->file_state() != BlobFileMeta::FileState::kInit);
+  if (blob_file != nullptr &&
+      blob_file->file_state() != BlobFileMeta::FileState::kNormal)
+    return false;
 
   return true;
 }

src/blob_gc_picker.h

@@ -29,7 +29,7 @@ class BlobGCPicker {
 
 class BasicBlobGCPicker final : public BlobGCPicker {
  public:
-  BasicBlobGCPicker(TitanDBOptions, TitanCFOptions);
+  BasicBlobGCPicker(TitanDBOptions, TitanCFOptions, TitanStats*);
   ~BasicBlobGCPicker();
 
   std::unique_ptr<BlobGC> PickBlobGC(BlobStorage* blob_storage) override;
@@ -37,6 +37,7 @@ class BasicBlobGCPicker final : public BlobGCPicker {
  private:
   TitanDBOptions db_options_;
   TitanCFOptions cf_options_;
+  TitanStats* stats_;
 
   // Check if blob_file needs to gc, return true means we need pick this
   // file for gc

src/blob_gc_picker_test.cc

@@ -26,7 +26,7 @@ class BlobGCPickerTest : public testing::Test {
     blob_storage_.reset(new BlobStorage(titan_db_options, titan_cf_options, 0,
                                         blob_file_cache, nullptr));
     basic_blob_gc_picker_.reset(
-        new BasicBlobGCPicker(titan_db_options, titan_cf_options));
+        new BasicBlobGCPicker(titan_db_options, titan_cf_options, nullptr));
   }
 
   void AddBlobFile(uint64_t file_number, uint64_t file_size,

src/blob_storage.h

@@ -68,6 +68,8 @@ class BlobStorage {
     for (auto& file : files_) {
       file.second->set_gc_mark(true);
       file.second->FileStateTransit(BlobFileMeta::FileEvent::kDbRestart);
+      auto level = file.second->GetDiscardableRatioLevel();
+      AddStats(stats_, cf_id_, level, 1);
     }
   }
 

src/db_impl.cc

@@ -58,7 +58,7 @@ class TitanDBImpl::FileManager : public BlobFileManager {
     for (auto& file : files) {
       RecordTick(db_->stats_.get(), BLOB_DB_BLOB_FILE_SYNCED);
       {
-        StopWatch sync_sw(db_->env_, statistics(db_->stats_.get()),
+        StopWatch sync_sw(db_->env_, db_->stats_.get(),
                           BLOB_DB_BLOB_FILE_SYNC_MICROS);
         s = file.second->GetFile()->Sync(false);
       }
@@ -546,7 +546,8 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options,
                         nullptr /*read_callback*/, &is_blob_index);
   if (!s.ok() || !is_blob_index) return s;
 
-  StopWatch get_sw(env_, statistics(stats_.get()), BLOB_DB_GET_MICROS);
+  StopWatch get_sw(env_, stats_.get(), BLOB_DB_GET_MICROS);
+  RecordTick(stats_.get(), BLOB_DB_NUM_GET);
 
   BlobIndex index;
   s = index.DecodeFrom(value);
@@ -561,8 +562,7 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options,
   mutex_.Unlock();
 
   {
-    StopWatch read_sw(env_, statistics(stats_.get()),
-                      BLOB_DB_BLOB_FILE_READ_MICROS);
+    StopWatch read_sw(env_, stats_.get(), BLOB_DB_BLOB_FILE_READ_MICROS);
     s = storage->Get(options, index, &record, &buffer);
     RecordTick(stats_.get(), BLOB_DB_NUM_KEYS_READ);
     RecordTick(stats_.get(), BLOB_DB_BLOB_FILE_BYTES_READ,
@@ -805,7 +805,13 @@ Status TitanDBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
     if (!file->is_obsolete()) {
       delta += bfs.second;
     }
+    auto before = file->GetDiscardableRatioLevel();
     file->AddDiscardableSize(static_cast<uint64_t>(bfs.second));
+    auto after = file->GetDiscardableRatioLevel();
+    if (before != after) {
+      AddStats(stats_.get(), cf_id, after, 1);
+      SubStats(stats_.get(), cf_id, before, 1);
+    }
   }
   SubStats(stats_.get(), cf_id, TitanInternalStats::LIVE_BLOB_SIZE, delta);
   bs->ComputeGCScore();
@@ -1093,7 +1099,13 @@ void TitanDBImpl::OnCompactionCompleted(
       if (!file->is_obsolete()) {
         delta += -bfs.second;
       }
+      auto before = file->GetDiscardableRatioLevel();
       file->AddDiscardableSize(static_cast<uint64_t>(-bfs.second));
+      auto after = file->GetDiscardableRatioLevel();
+      if (before != after) {
+        AddStats(stats_.get(), compaction_job_info.cf_id, after, 1);
+        SubStats(stats_.get(), compaction_job_info.cf_id, before, 1);
+      }
     }
     SubStats(stats_.get(), compaction_job_info.cf_id,
              TitanInternalStats::LIVE_BLOB_SIZE, delta);

src/db_impl_gc.cc

@@ -69,7 +69,7 @@ void TitanDBImpl::BackgroundCallGC() {
 
 Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer) {
   mutex_.AssertHeld();
-  StopWatch gc_sw(env_, statistics(stats_.get()), BLOB_DB_GC_MICROS);
+  StopWatch gc_sw(env_, stats_.get(), BLOB_DB_GC_MICROS);
 
   std::unique_ptr<BlobGC> blob_gc;
   std::unique_ptr<ColumnFamilyHandle> cfh;
@@ -88,7 +88,8 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer) {
     if (blob_storage != nullptr) {
       const auto& cf_options = blob_storage->cf_options();
       std::shared_ptr<BlobGCPicker> blob_gc_picker =
-          std::make_shared<BasicBlobGCPicker>(db_options_, cf_options);
+          std::make_shared<BasicBlobGCPicker>(db_options_, cf_options,
+                                              stats_.get());
       blob_gc = blob_gc_picker->PickBlobGC(blob_storage.get());
 
       if (blob_gc) {
@@ -123,6 +124,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer) {
     if (blob_gc->trigger_next() &&
         (bg_gc_scheduled_ - 1 + gc_queue_.size() <
          2 * static_cast<uint32_t>(db_options_.max_background_gc))) {
+      RecordTick(stats_.get(), TitanStats::GC_TRIGGER_NEXT, 1);
       // There is still data remained to be GCed
       // and the queue is not overwhelmed
       // then put this cf to GC queue for next GC
@@ -131,9 +133,11 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer) {
   }
 
   if (s.ok()) {
+    RecordTick(stats_.get(), TitanStats::GC_SUCCESS, 1);
     // Done
   } else {
     SetBGError(s);
+    RecordTick(stats_.get(), TitanStats::GC_FAIL, 1);
     ROCKS_LOG_WARN(db_options_.info_log, "Titan GC error: %s",
                    s.ToString().c_str());
   }
@@ -157,7 +161,7 @@ Status TitanDBImpl::TEST_StartGC(uint32_t column_family_id) {
     bg_gc_scheduled_++;
 
     // BackgroudGC
-    StopWatch gc_sw(env_, statistics(stats_.get()), BLOB_DB_GC_MICROS);
+    StopWatch gc_sw(env_, stats_.get(), BLOB_DB_GC_MICROS);
 
     std::unique_ptr<BlobGC> blob_gc;
     std::unique_ptr<ColumnFamilyHandle> cfh;
@@ -169,7 +173,7 @@ Status TitanDBImpl::TEST_StartGC(uint32_t column_family_id) {
     auto bs = blob_file_set_->GetBlobStorage(column_family_id).lock().get();
     const auto& cf_options = bs->cf_options();
     std::shared_ptr<BlobGCPicker> blob_gc_picker =
-        std::make_shared<BasicBlobGCPicker>(db_options_, cf_options);
+        std::make_shared<BasicBlobGCPicker>(db_options_, cf_options, nullptr);
     blob_gc = blob_gc_picker->PickBlobGC(bs);
 
     if (blob_gc) {

src/db_iter.h

@@ -46,7 +46,7 @@ class TitanDBIterator : public Iterator {
   void SeekToFirst() override {
     iter_->SeekToFirst();
     if (ShouldGetBlobValue()) {
-      StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS);
+      StopWatch seek_sw(env_, stats_, BLOB_DB_SEEK_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_SEEK);
     }
@@ -55,7 +55,7 @@ class TitanDBIterator : public Iterator {
   void SeekToLast() override {
     iter_->SeekToLast();
     if (ShouldGetBlobValue()) {
-      StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS);
+      StopWatch seek_sw(env_, stats_, BLOB_DB_SEEK_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_SEEK);
     }
@@ -64,7 +64,7 @@ class TitanDBIterator : public Iterator {
   void Seek(const Slice& target) override {
     iter_->Seek(target);
     if (ShouldGetBlobValue()) {
-      StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS);
+      StopWatch seek_sw(env_, stats_, BLOB_DB_SEEK_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_SEEK);
     }
@@ -73,7 +73,7 @@ class TitanDBIterator : public Iterator {
   void SeekForPrev(const Slice& target) override {
     iter_->SeekForPrev(target);
     if (ShouldGetBlobValue()) {
-      StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS);
+      StopWatch seek_sw(env_, stats_, BLOB_DB_SEEK_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_SEEK);
     }
@@ -83,7 +83,7 @@ class TitanDBIterator : public Iterator {
     assert(Valid());
     iter_->Next();
     if (ShouldGetBlobValue()) {
-      StopWatch next_sw(env_, statistics(stats_), BLOB_DB_NEXT_MICROS);
+      StopWatch next_sw(env_, stats_, BLOB_DB_NEXT_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_NEXT);
     }
@@ -93,7 +93,7 @@ class TitanDBIterator : public Iterator {
     assert(Valid());
     iter_->Prev();
     if (ShouldGetBlobValue()) {
-      StopWatch prev_sw(env_, statistics(stats_), BLOB_DB_PREV_MICROS);
+      StopWatch prev_sw(env_, stats_, BLOB_DB_PREV_MICROS);
       GetBlobValue();
       RecordTick(stats_, BLOB_DB_NUM_PREV);
     }

src/table_builder.cc

@@ -5,6 +5,7 @@
 #endif
 
 #include <inttypes.h>
+#include "monitoring/statistics.h"
 
 namespace rocksdb {
 namespace titandb {
@@ -106,8 +107,7 @@ void TitanTableBuilder::Add(const Slice& key, const Slice& value) {
 void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value,
                                 std::string* index_value) {
   if (!ok()) return;
-  StopWatch write_sw(db_options_.env, statistics(stats_),
-                     BLOB_DB_BLOB_FILE_WRITE_MICROS);
+  StopWatch write_sw(db_options_.env, stats_, BLOB_DB_BLOB_FILE_WRITE_MICROS);
 
   if (!blob_builder_) {
     status_ = blob_manager_->NewFile(&blob_handle_);
@@ -120,8 +120,8 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value,
   }
 
   RecordTick(stats_, BLOB_DB_NUM_KEYS_WRITTEN);
-  MeasureTime(stats_, BLOB_DB_KEY_SIZE, key.size());
-  MeasureTime(stats_, BLOB_DB_VALUE_SIZE, value.size());
+  RecordInHistogram(stats_, BLOB_DB_KEY_SIZE, key.size());
+  RecordInHistogram(stats_, BLOB_DB_VALUE_SIZE, value.size());
   AddStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_SIZE, value.size());
   bytes_written_ += key.size() + value.size();
 

src/titan_stats.cc

@@ -14,6 +14,16 @@ static const std::string num_live_blob_file = "num-live-blob-file";
 static const std::string num_obsolete_blob_file = "num-obsolete-blob-file";
 static const std::string live_blob_file_size = "live-blob-file-size";
 static const std::string obsolete_blob_file_size = "obsolete-blob-file-size";
+static const std::string num_discardable_ratio_le0_file =
+    "num-discardable-ratio-le0-file";
+static const std::string num_discardable_ratio_le20_file =
+    "num-discardable-ratio-le20-file";
+static const std::string num_discardable_ratio_le50_file =
+    "num-discardable-ratio-le50-file";
+static const std::string num_discardable_ratio_le80_file =
+    "num-discardable-ratio-le80-file";
+static const std::string num_discardable_ratio_le100_file =
+    "num-discardable-ratio-le100-file";
 
 const std::string TitanDB::Properties::kLiveBlobSize =
     titandb_prefix + live_blob_size;
@@ -25,9 +35,19 @@ const std::string TitanDB::Properties::kLiveBlobFileSize =
     titandb_prefix + live_blob_file_size;
 const std::string TitanDB::Properties::kObsoleteBlobFileSize =
     titandb_prefix + obsolete_blob_file_size;
+const std::string TitanDB::Properties::kNumDiscardableRatioLE0File =
+    titandb_prefix + num_discardable_ratio_le0_file;
+const std::string TitanDB::Properties::kNumDiscardableRatioLE20File =
+    titandb_prefix + num_discardable_ratio_le20_file;
+const std::string TitanDB::Properties::kNumDiscardableRatioLE50File =
+    titandb_prefix + num_discardable_ratio_le50_file;
+const std::string TitanDB::Properties::kNumDiscardableRatioLE80File =
+    titandb_prefix + num_discardable_ratio_le80_file;
+const std::string TitanDB::Properties::kNumDiscardableRatioLE100File =
+    titandb_prefix + num_discardable_ratio_le100_file;
 
-const std::unordered_map<std::string, TitanInternalStats::TickerType>
-    TitanInternalStats::ticker_type_string_map = {
+const std::unordered_map<std::string, TitanInternalStats::StatsType>
+    TitanInternalStats::stats_type_string_map = {
         {TitanDB::Properties::kLiveBlobSize,
          TitanInternalStats::LIVE_BLOB_SIZE},
         {TitanDB::Properties::kNumLiveBlobFile,
@@ -38,11 +58,18 @@ const std::unordered_map<std::string, TitanInternalStats::TickerType>
          TitanInternalStats::LIVE_BLOB_FILE_SIZE},
         {TitanDB::Properties::kObsoleteBlobFileSize,
          TitanInternalStats::OBSOLETE_BLOB_FILE_SIZE},
+        {TitanDB::Properties::kNumDiscardableRatioLE0File,
+         TitanInternalStats::NUM_DISCARDABLE_RATIO_LE0},
+        {TitanDB::Properties::kNumDiscardableRatioLE20File,
+         TitanInternalStats::NUM_DISCARDABLE_RATIO_LE20},
+        {TitanDB::Properties::kNumDiscardableRatioLE50File,
+         TitanInternalStats::NUM_DISCARDABLE_RATIO_LE50},
+        {TitanDB::Properties::kNumDiscardableRatioLE80File,
+         TitanInternalStats::NUM_DISCARDABLE_RATIO_LE80},
+        {TitanDB::Properties::kNumDiscardableRatioLE100File,
+         TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100},
 };
 
-const std::unordered_map<std::string, TitanInternalStats::HistogramType>
-    TitanInternalStats::histogram_type_string_map = {};
-
 const std::array<std::string,
                  static_cast<int>(InternalOpType::INTERNAL_OP_ENUM_MAX)>
     TitanInternalStats::internal_op_names = {

src/util.cc

 #include "util.h"
 
+#include "util/stop_watch.h"
+
 namespace rocksdb {
 namespace titandb {
 
@@ -157,5 +159,12 @@ void UnrefCacheHandle(void* arg1, void* arg2) {
   cache->Release(h);
 }
 
+Status SyncTitanManifest(Env* env, TitanStats* stats,
+                         const ImmutableDBOptions* db_options,
+                         WritableFileWriter* file) {
+  StopWatch sw(env, stats, TitanStats::TITAN_MANIFEST_FILE_SYNC_MICROS);
+  return file->Sync(db_options->use_fsync);
+}
+
 }  // namespace titandb
 }  // namespace rocksdb

src/util.h

 #pragma once
 
+#include "options/db_options.h"
 #include "rocksdb/cache.h"
 #include "util/compression.h"
+#include "util/file_reader_writer.h"
+
+#include "titan_stats.h"
 
 namespace rocksdb {
 namespace titandb {
@@ -71,5 +75,9 @@ void DeleteCacheValue(const Slice&, void* value) {
   delete reinterpret_cast<T*>(value);
 }
 
+Status SyncTitanManifest(Env* env, TitanStats* stats,
+                         const ImmutableDBOptions* db_options,
+                         WritableFileWriter* file);
+
 }  // namespace titandb
 }  // namespace rocksdb