exclude metadata size from file size for discardable ratio (#154)

exclude metadata size from file size, otherwise discardable ratio would be wrong when the blob file is small.
Signed-off-by: Connor <zbk602423539@gmail.com>

src/blob_format.cc

@@ -290,8 +290,9 @@ TitanInternalStats::StatsType BlobFileMeta::GetDiscardableRatioLevel() const {
              (ratio - 1.0) < std::numeric_limits<double>::epsilon()) {
     type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100;
   } else {
-    fprintf(stderr, "invalid discardable ratio");
-    abort();
+    fprintf(stderr, "invalid discardable ratio  %lf for blob file %" PRIu64,
+            ratio, this->file_number_);
+    type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100;
   }
   return type;
 }

src/blob_format.h

@@ -259,8 +259,9 @@ class BlobFileMeta {
     if (file_size_ == 0) {
       return 0;
     }
-    // TODO: Exclude metadata size from file size.
-    return 1 - (static_cast<double>(live_data_size_) / file_size_);
+    // TODO: Exclude meta blocks from file size
+    return 1 - (static_cast<double>(live_data_size_) /
+                (file_size_ - kBlobMaxHeaderSize - kBlobFooterSize));
   }
   TitanInternalStats::StatsType GetDiscardableRatioLevel() const;
 

src/blob_gc_picker_test.cc

@@ -29,11 +29,12 @@ class BlobGCPickerTest : public testing::Test {
         new BasicBlobGCPicker(titan_db_options, titan_cf_options, nullptr));
   }
 
-  void AddBlobFile(uint64_t file_number, uint64_t file_size,
+  void AddBlobFile(uint64_t file_number, uint64_t data_size,
                    uint64_t discardable_size, bool being_gc = false) {
-    auto f =
-        std::make_shared<BlobFileMeta>(file_number, file_size, 0, 0, "", "");
-    f->set_live_data_size(file_size - discardable_size);
+    auto f = std::make_shared<BlobFileMeta>(
+        file_number, data_size + kBlobMaxHeaderSize + kBlobFooterSize, 0, 0, "",
+        "");
+    f->set_live_data_size(data_size - discardable_size);
     f->FileStateTransit(BlobFileMeta::FileEvent::kDbRestart);
     if (being_gc) {
       f->FileStateTransit(BlobFileMeta::FileEvent::kGCBegin);

src/titan_db_test.cc

@@ -1267,6 +1267,17 @@ TEST_F(TitanDBTest, GCAfterReopen) {
     ASSERT_OK(db_->Put(WriteOptions(), key, "v"));
   }
   Flush();
+
+  std::shared_ptr<BlobStorage> blob_storage = GetBlobStorage().lock();
+  ASSERT_TRUE(blob_storage != nullptr);
+  std::map<uint64_t, std::weak_ptr<BlobFileMeta>> blob_files;
+  blob_storage->ExportBlobFiles(blob_files);
+  ASSERT_EQ(1, blob_files.size());
+  std::shared_ptr<BlobFileMeta> file1 = blob_files.begin()->second.lock();
+  ASSERT_TRUE(file1 != nullptr);
+  ASSERT_EQ(file1->GetDiscardableRatioLevel(),
+            TitanInternalStats::NUM_DISCARDABLE_RATIO_LE0);
+
   for (int i = 0; i < 100; i++) {
     if (i % 2 == 0) {
       Delete(i);
@@ -1274,13 +1285,8 @@ TEST_F(TitanDBTest, GCAfterReopen) {
   }
   Flush();
   CompactAll();
-  std::shared_ptr<BlobStorage> blob_storage = GetBlobStorage().lock();
-  ASSERT_TRUE(blob_storage != nullptr);
-  std::map<uint64_t, std::weak_ptr<BlobFileMeta>> blob_files;
   blob_storage->ExportBlobFiles(blob_files);
   ASSERT_EQ(1, blob_files.size());
-  std::shared_ptr<BlobFileMeta> file1 = blob_files.begin()->second.lock();
-  ASSERT_TRUE(file1 != nullptr);
   ASSERT_TRUE(abs(file1->GetDiscardableRatio() - 0.5) < 0.01);
   uint64_t file_number1 = file1->file_number();
   file1.reset();