Skip to content

T
titan
Commit e6fa07d5 authored by Connor's avatar Connor Committed by yiwu-arbug
Browse files
Options

Export Titan customized statistics (#129) 

Signed-off-by: 's avatarConnor1996 <zbk602423539@gmail.com>
parent 423ce1f7
Hide whitespace changes
Inline Side-by-side
Showing with 193 additions and 84 deletions
include/titan/statistics.h 0 → 100644
View file @ e6fa07d5
#pragma once
#include "rocksdb/statistics.h"
namespace rocksdb {
namespace titandb {
std::shared_ptr<Statistics> CreateDBStatistics();
enum TickerType : uint32_t {
TITAN_NUM_GET = TICKER_ENUM_MAX,
TITAN_NUM_SEEK,
TITAN_NUM_NEXT,
TITAN_NUM_PREV,
TITAN_BLOB_FILE_NUM_KEYS_WRITTEN,
TITAN_BLOB_FILE_NUM_KEYS_READ,
TITAN_BLOB_FILE_BYTES_WRITTEN,
TITAN_BLOB_FILE_BYTES_READ,
TITAN_BLOB_FILE_SYNCED,
TITAN_GC_NUM_FILES,
TITAN_GC_NUM_NEW_FILES,
TITAN_GC_NUM_KEYS_OVERWRITTEN,
TITAN_GC_NUM_KEYS_RELOCATED,
TITAN_GC_BYTES_OVERWRITTEN,
TITAN_GC_BYTES_RELOCATED,
TITAN_GC_BYTES_WRITTEN,
TITAN_GC_BYTES_READ,
TITAN_BLOB_CACHE_HIT,
TITAN_BLOB_CACHE_MISS,
TITAN_GC_NO_NEED,
TITAN_GC_REMAIN,
TITAN_GC_DISCARDABLE,
TITAN_GC_SAMPLE,
TITAN_GC_SMALL_FILE,
TITAN_GC_FAILURE,
TITAN_GC_SUCCESS,
TITAN_GC_TRIGGER_NEXT,
TITAN_TICKER_ENUM_MAX,
};
// The order of items listed in Tickers should be the same as
// the order listed in TickersNameMap
const std::vector<std::pair<TickerType, std::string>> TitanTickersNameMap = {
{TITAN_NUM_GET, "titandb.num.get"},
{TITAN_NUM_SEEK, "titandb.num.seek"},
{TITAN_NUM_NEXT, "titandb.num.next"},
{TITAN_NUM_PREV, "titandb.num.prev"},
{TITAN_BLOB_FILE_NUM_KEYS_WRITTEN, "titandb.blob.file.num.keys.written"},
{TITAN_BLOB_FILE_NUM_KEYS_READ, "titandb.blob.file.num.keys.read"},
{TITAN_BLOB_FILE_BYTES_WRITTEN, "titandb.blob.file.bytes.written"},
{TITAN_BLOB_FILE_BYTES_READ, "titandb.blob.file.bytes.read"},
{TITAN_BLOB_FILE_SYNCED, "titandb.blob.file.synced"},
{TITAN_GC_NUM_FILES, "titandb.gc.num.files"},
{TITAN_GC_NUM_NEW_FILES, "titandb.gc.num.new.files"},
{TITAN_GC_NUM_KEYS_OVERWRITTEN, "titandb.gc.num.keys.overwritten"},
{TITAN_GC_NUM_KEYS_RELOCATED, "titandb.gc.num.keys.relocated"},
{TITAN_GC_BYTES_OVERWRITTEN, "titandb.gc.bytes.overwritten"},
{TITAN_GC_BYTES_RELOCATED, "titandb.gc.bytes.relocated"},
{TITAN_GC_BYTES_WRITTEN, "titandb.gc.bytes.written"},
{TITAN_GC_BYTES_READ, "titandb.gc.bytes.read"},
{TITAN_BLOB_CACHE_HIT, "titandb.blob.cache.hit"},
{TITAN_BLOB_CACHE_MISS, "titandb.blob.cache.miss"},
{TITAN_GC_NO_NEED, "titandb.gc.no.need"},
{TITAN_GC_REMAIN, "titandb.gc.remain"},
{TITAN_GC_DISCARDABLE, "titandb.gc.discardable"},
{TITAN_GC_SAMPLE, "titandb.gc.sample"},
{TITAN_GC_SMALL_FILE, "titandb.gc.small.file"},
{TITAN_GC_FAILURE, "titandb.gc.failure"},
{TITAN_GC_SUCCESS, "titandb.gc.success"},
{TITAN_GC_TRIGGER_NEXT, "titandb.gc.trigger.next"},
};
enum HistogramType : uint32_t {
TITAN_KEY_SIZE = HISTOGRAM_ENUM_MAX,
TITAN_VALUE_SIZE,
TITAN_GET_MICROS,
TITAN_SEEK_MICROS,
TITAN_NEXT_MICROS,
TITAN_PREV_MICROS,
TITAN_BLOB_FILE_WRITE_MICROS,
TITAN_BLOB_FILE_READ_MICROS,
TITAN_BLOB_FILE_SYNC_MICROS,
TITAN_MANIFEST_FILE_SYNC_MICROS,
TITAN_GC_MICROS,
TITAN_GC_INPUT_FILE_SIZE,
TITAN_GC_OUTPUT_FILE_SIZE,
TITAN_ITER_TOUCH_BLOB_FILE_COUNT,
TITAN_HISTOGRAM_ENUM_MAX,
};
const std::vector<std::pair<HistogramType, std::string>>
TitanHistogramsNameMap = {
{TITAN_KEY_SIZE, "titandb.key.size"},
{TITAN_VALUE_SIZE, "titandb.value.size"},
{TITAN_GET_MICROS, "titandb.get.micros"},
{TITAN_SEEK_MICROS, "titandb.seek.micros"},
{TITAN_NEXT_MICROS, "titandb.next.micros"},
{TITAN_PREV_MICROS, "titandb.prev.micros"},
{TITAN_BLOB_FILE_WRITE_MICROS, "titandb.blob.file.write.micros"},
{TITAN_BLOB_FILE_READ_MICROS, "titandb.blob.file.read.micros"},
{TITAN_BLOB_FILE_SYNC_MICROS, "titandb.blob.file.sync.micros"},
{TITAN_MANIFEST_FILE_SYNC_MICROS, "titandb.manifest.file.sync.micros"},
{TITAN_GC_MICROS, "titandb.gc.micros"},
{TITAN_GC_INPUT_FILE_SIZE, "titandb.gc.input.file.size"},
{TITAN_GC_OUTPUT_FILE_SIZE, "titandb.gc.output.file.size"},
{TITAN_ITER_TOUCH_BLOB_FILE_COUNT,
"titandb.iter.touch.blob.file.count"},
};
} // namespace titandb
} // namespace rocksdb
src/blob_file_reader.cc
View file @ e6fa07d5
...@@ -106,13 +106,13 @@ Status BlobFileReader::Get(const ReadOptions& /*options*/, ...@@ -106,13 +106,13 @@ Status BlobFileReader::Get(const ReadOptions& /*options*/,
EncodeBlobCache(&cache_key, cache_prefix_, handle.offset); EncodeBlobCache(&cache_key, cache_prefix_, handle.offset);
cache_handle = cache_->Lookup(cache_key); cache_handle = cache_->Lookup(cache_key);
if (cache_handle) { if (cache_handle) {
RecordTick(statistics(stats_), BLOB_CACHE_HIT); RecordTick(statistics(stats_), TITAN_BLOB_CACHE_HIT);
auto blob = reinterpret_cast<OwnedSlice*>(cache_->Value(cache_handle)); auto blob = reinterpret_cast<OwnedSlice*>(cache_->Value(cache_handle));
buffer->PinSlice(*blob, UnrefCacheHandle, cache_.get(), cache_handle); buffer->PinSlice(*blob, UnrefCacheHandle, cache_.get(), cache_handle);
return DecodeInto(*blob, record); return DecodeInto(*blob, record);
} }
} }
RecordTick(statistics(stats_), BLOB_CACHE_MISS); RecordTick(statistics(stats_), TITAN_BLOB_CACHE_MISS);
OwnedSlice blob; OwnedSlice blob;
Status s = ReadRecord(handle, record, &blob); Status s = ReadRecord(handle, record, &blob);
......
src/blob_format.cc
View file @ e6fa07d5
...@@ -290,7 +290,7 @@ TitanInternalStats::StatsType BlobFileMeta::GetDiscardableRatioLevel() const { ...@@ -290,7 +290,7 @@ TitanInternalStats::StatsType BlobFileMeta::GetDiscardableRatioLevel() const {
(ratio - 1.0) < std::numeric_limits<double>::epsilon()) { (ratio - 1.0) < std::numeric_limits<double>::epsilon()) {
type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100; type = TitanInternalStats::NUM_DISCARDABLE_RATIO_LE100;
} else { } else {
fprintf(stderr, "invalid discarable ratio"); fprintf(stderr, "invalid discardable ratio");
abort(); abort();
} }
return type; return type;
......
src/blob_format.h
View file @ e6fa07d5
...@@ -256,6 +256,9 @@ class BlobFileMeta { ...@@ -256,6 +256,9 @@ class BlobFileMeta {
} }
bool NoLiveData() { return live_data_size_ == 0; } bool NoLiveData() { return live_data_size_ == 0; }
double GetDiscardableRatio() const { double GetDiscardableRatio() const {
if (file_size_ == 0) {
return 0;
}
// TODO: Exclude metadata size from file size. // TODO: Exclude metadata size from file size.
return 1 - (static_cast<double>(live_data_size_) / file_size_); return 1 - (static_cast<double>(live_data_size_) / file_size_);
} }
......
src/blob_gc_job.cc
View file @ e6fa07d5
...@@ -101,22 +101,23 @@ BlobGCJob::~BlobGCJob() { ...@@ -101,22 +101,23 @@ BlobGCJob::~BlobGCJob() {
LogFlush(db_options_.info_log.get()); LogFlush(db_options_.info_log.get());
} }
// flush metrics // flush metrics
RecordTick(statistics(stats_), BLOB_DB_BYTES_READ, metrics_.bytes_read); RecordTick(statistics(stats_), TITAN_GC_BYTES_READ, metrics_.gc_bytes_read);
RecordTick(statistics(stats_), BLOB_DB_BYTES_WRITTEN, metrics_.bytes_written); RecordTick(statistics(stats_), TITAN_GC_BYTES_WRITTEN,
RecordTick(statistics(stats_), BLOB_DB_GC_NUM_KEYS_OVERWRITTEN, metrics_.gc_bytes_written);
RecordTick(statistics(stats_), TITAN_GC_NUM_KEYS_OVERWRITTEN,
metrics_.gc_num_keys_overwritten); metrics_.gc_num_keys_overwritten);
RecordTick(statistics(stats_), BLOB_DB_GC_BYTES_OVERWRITTEN, RecordTick(statistics(stats_), TITAN_GC_BYTES_OVERWRITTEN,
metrics_.gc_bytes_overwritten); metrics_.gc_bytes_overwritten);
RecordTick(statistics(stats_), BLOB_DB_GC_NUM_KEYS_RELOCATED, RecordTick(statistics(stats_), TITAN_GC_NUM_KEYS_RELOCATED,
metrics_.gc_num_keys_relocated); metrics_.gc_num_keys_relocated);
RecordTick(statistics(stats_), BLOB_DB_GC_BYTES_RELOCATED, RecordTick(statistics(stats_), TITAN_GC_BYTES_RELOCATED,
metrics_.gc_bytes_relocated); metrics_.gc_bytes_relocated);
RecordTick(statistics(stats_), BLOB_DB_GC_NUM_NEW_FILES, RecordTick(statistics(stats_), TITAN_GC_NUM_NEW_FILES,
metrics_.gc_num_new_files); metrics_.gc_num_new_files);
RecordTick(statistics(stats_), BLOB_DB_GC_NUM_FILES, metrics_.gc_num_files); RecordTick(statistics(stats_), TITAN_GC_NUM_FILES, metrics_.gc_num_files);
RecordTick(statistics(stats_), GC_DISCARDABLE, metrics_.gc_discardable); RecordTick(statistics(stats_), TITAN_GC_DISCARDABLE, metrics_.gc_discardable);
RecordTick(statistics(stats_), GC_SMALL_FILE, metrics_.gc_small_file); RecordTick(statistics(stats_), TITAN_GC_SMALL_FILE, metrics_.gc_small_file);
RecordTick(statistics(stats_), GC_SAMPLE, metrics_.gc_sample); RecordTick(statistics(stats_), TITAN_GC_SAMPLE, metrics_.gc_sample);
} }
Status BlobGCJob::Prepare() { Status BlobGCJob::Prepare() {
...@@ -177,7 +178,7 @@ Status BlobGCJob::DoRunGC() { ...@@ -177,7 +178,7 @@ Status BlobGCJob::DoRunGC() {
} }
BlobIndex blob_index = gc_iter->GetBlobIndex(); BlobIndex blob_index = gc_iter->GetBlobIndex();
// count read bytes for blob record of gc candidate files // count read bytes for blob record of gc candidate files
metrics_.bytes_read += blob_index.blob_handle.size; metrics_.gc_bytes_read += blob_index.blob_handle.size;
if (!last_key.empty() && !gc_iter->key().compare(last_key)) { if (!last_key.empty() && !gc_iter->key().compare(last_key)) {
if (last_key_valid) { if (last_key_valid) {
...@@ -231,7 +232,7 @@ Status BlobGCJob::DoRunGC() { ...@@ -231,7 +232,7 @@ Status BlobGCJob::DoRunGC() {
blob_record.value = gc_iter->value(); blob_record.value = gc_iter->value();
// count written bytes for new blob record, // count written bytes for new blob record,
// blob index's size is counted in `RewriteValidKeyToLSM` // blob index's size is counted in `RewriteValidKeyToLSM`
metrics_.bytes_written += blob_record.size(); metrics_.gc_bytes_written += blob_record.size();
MergeBlobIndex new_blob_index; MergeBlobIndex new_blob_index;
new_blob_index.file_number = blob_file_handle->GetNumber(); new_blob_index.file_number = blob_file_handle->GetNumber();
...@@ -319,7 +320,7 @@ Status BlobGCJob::DiscardEntry(const Slice& key, const BlobIndex& blob_index, ...@@ -319,7 +320,7 @@ Status BlobGCJob::DiscardEntry(const Slice& key, const BlobIndex& blob_index,
return s; return s;
} }
// count read bytes for checking LSM entry // count read bytes for checking LSM entry
metrics_.bytes_read += key.size() + index_entry.size(); metrics_.gc_bytes_read += key.size() + index_entry.size();
if (s.IsNotFound() || !is_blob_index) { if (s.IsNotFound() || !is_blob_index) {
// Either the key is deleted or updated with a newer version which is // Either the key is deleted or updated with a newer version which is
// inlined in LSM. // inlined in LSM.
...@@ -396,7 +397,7 @@ Status BlobGCJob::InstallOutputBlobFiles() { ...@@ -396,7 +397,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
builder.second->GetLargestKey()); builder.second->GetLargestKey());
file->set_live_data_size(builder.second->live_data_size()); file->set_live_data_size(builder.second->live_data_size());
file->FileStateTransit(BlobFileMeta::FileEvent::kGCOutput); file->FileStateTransit(BlobFileMeta::FileEvent::kGCOutput);
RecordInHistogram(statistics(stats_), GC_OUTPUT_FILE_SIZE, RecordInHistogram(statistics(stats_), TITAN_GC_OUTPUT_FILE_SIZE,
file->file_size()); file->file_size());
if (!tmp.empty()) { if (!tmp.empty()) {
tmp.append(" "); tmp.append(" ");
...@@ -464,7 +465,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() { ...@@ -464,7 +465,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
&write_batch.second); &write_batch.second);
if (s.ok()) { if (s.ok()) {
// count written bytes for new blob index. // count written bytes for new blob index.
metrics_.bytes_written += write_batch.first.GetDataSize(); metrics_.gc_bytes_written += write_batch.first.GetDataSize();
metrics_.gc_num_keys_relocated++; metrics_.gc_num_keys_relocated++;
metrics_.gc_bytes_relocated += write_batch.second.blob_record_size(); metrics_.gc_bytes_relocated += write_batch.second.blob_record_size();
// Key is successfully written to LSM. // Key is successfully written to LSM.
...@@ -477,7 +478,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() { ...@@ -477,7 +478,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
break; break;
} }
// count read bytes in write callback // count read bytes in write callback
metrics_.bytes_read += write_batch.second.read_bytes(); metrics_.gc_bytes_read += write_batch.second.read_bytes();
} }
} else { } else {
for (auto& write_batch : rewrite_batches_without_callback_) { for (auto& write_batch : rewrite_batches_without_callback_) {
...@@ -492,7 +493,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() { ...@@ -492,7 +493,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
s = db_impl->Write(wo, &write_batch.first); s = db_impl->Write(wo, &write_batch.first);
if (s.ok()) { if (s.ok()) {
// count written bytes for new blob index. // count written bytes for new blob index.
metrics_.bytes_written += write_batch.first.GetDataSize(); metrics_.gc_bytes_written += write_batch.first.GetDataSize();
metrics_.gc_num_keys_relocated++; metrics_.gc_num_keys_relocated++;
metrics_.gc_bytes_relocated += write_batch.second; metrics_.gc_bytes_relocated += write_batch.second;
// Key is successfully written to LSM. // Key is successfully written to LSM.
...@@ -528,7 +529,7 @@ Status BlobGCJob::DeleteInputBlobFiles() { ...@@ -528,7 +529,7 @@ Status BlobGCJob::DeleteInputBlobFiles() {
Slice(file->smallest_key()).ToString(true).c_str(), Slice(file->smallest_key()).ToString(true).c_str(),
Slice(file->largest_key()).ToString(true).c_str()); Slice(file->largest_key()).ToString(true).c_str());
metrics_.gc_num_files++; metrics_.gc_num_files++;
RecordInHistogram(statistics(stats_), GC_INPUT_FILE_SIZE, RecordInHistogram(statistics(stats_), TITAN_GC_INPUT_FILE_SIZE,
file->file_size()); file->file_size());
edit.DeleteBlobFile(file->file_number(), obsolete_sequence); edit.DeleteBlobFile(file->file_number(), obsolete_sequence);
} }
...@@ -558,9 +559,9 @@ void BlobGCJob::UpdateInternalOpStats() { ...@@ -558,9 +559,9 @@ void BlobGCJob::UpdateInternalOpStats() {
assert(internal_op_stats != nullptr); assert(internal_op_stats != nullptr);
AddStats(internal_op_stats, InternalOpStatsType::COUNT); AddStats(internal_op_stats, InternalOpStatsType::COUNT);
AddStats(internal_op_stats, InternalOpStatsType::BYTES_READ, AddStats(internal_op_stats, InternalOpStatsType::BYTES_READ,
metrics_.bytes_read); metrics_.gc_bytes_read);
AddStats(internal_op_stats, InternalOpStatsType::BYTES_WRITTEN, AddStats(internal_op_stats, InternalOpStatsType::BYTES_WRITTEN,
metrics_.bytes_written); metrics_.gc_bytes_written);
AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_READ, AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_READ,
io_bytes_read_); io_bytes_read_);
AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_WRITTEN, AddStats(internal_op_stats, InternalOpStatsType::IO_BYTES_WRITTEN,
......
src/blob_gc_job.h
View file @ e6fa07d5
...@@ -68,8 +68,8 @@ class BlobGCJob { ...@@ -68,8 +68,8 @@ class BlobGCJob {
TitanStats *stats_; TitanStats *stats_;
struct { struct {
uint64_t bytes_read = 0; uint64_t gc_bytes_read = 0;
uint64_t bytes_written = 0; uint64_t gc_bytes_written = 0;
uint64_t gc_num_keys_overwritten = 0; uint64_t gc_num_keys_overwritten = 0;
uint64_t gc_bytes_overwritten = 0; uint64_t gc_bytes_overwritten = 0;
uint64_t gc_num_keys_relocated = 0; uint64_t gc_num_keys_relocated = 0;
......
src/blob_gc_picker.cc
View file @ e6fa07d5
...@@ -32,7 +32,6 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC( ...@@ -32,7 +32,6 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
} }
auto blob_file = blob_storage->FindFile(gc_score.file_number).lock(); auto blob_file = blob_storage->FindFile(gc_score.file_number).lock();
if (!CheckBlobFile(blob_file.get())) { if (!CheckBlobFile(blob_file.get())) {
RecordTick(statistics(stats_), GC_NO_NEED, 1);
// Skip this file id this file is being GCed // Skip this file id this file is being GCed
// or this file had been GCed // or this file had been GCed
ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 " no need gc", ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 " no need gc",
...@@ -53,7 +52,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC( ...@@ -53,7 +52,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
next_gc_size += blob_file->file_size(); next_gc_size += blob_file->file_size();
if (next_gc_size > cf_options_.min_gc_batch_size) { if (next_gc_size > cf_options_.min_gc_batch_size) {
maybe_continue_next_time = true; maybe_continue_next_time = true;
RecordTick(statistics(stats_), GC_REMAIN, 1); RecordTick(statistics(stats_), TITAN_GC_REMAIN, 1);
ROCKS_LOG_INFO(db_options_.info_log, ROCKS_LOG_INFO(db_options_.info_log,
"remain more than %" PRIu64 "remain more than %" PRIu64
" bytes to be gc and trigger after this gc", " bytes to be gc and trigger after this gc",
......
src/blob_storage.cc
View file @ e6fa07d5
...@@ -90,6 +90,7 @@ void BlobStorage::AddBlobFile(std::shared_ptr<BlobFileMeta>& file) { ...@@ -90,6 +90,7 @@ void BlobStorage::AddBlobFile(std::shared_ptr<BlobFileMeta>& file) {
MutexLock l(&mutex_); MutexLock l(&mutex_);
files_.emplace(std::make_pair(file->file_number(), file)); files_.emplace(std::make_pair(file->file_number(), file));
blob_ranges_.emplace(std::make_pair(Slice(file->smallest_key()), file)); blob_ranges_.emplace(std::make_pair(Slice(file->smallest_key()), file));
AddStats(stats_, cf_id_, file->GetDiscardableRatioLevel(), 1);
AddStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_FILE_SIZE, AddStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_FILE_SIZE,
file->file_size()); file->file_size());
AddStats(stats_, cf_id_, TitanInternalStats::NUM_LIVE_BLOB_FILE, 1); AddStats(stats_, cf_id_, TitanInternalStats::NUM_LIVE_BLOB_FILE, 1);
...@@ -113,6 +114,7 @@ void BlobStorage::MarkFileObsoleteLocked(std::shared_ptr<BlobFileMeta> file, ...@@ -113,6 +114,7 @@ void BlobStorage::MarkFileObsoleteLocked(std::shared_ptr<BlobFileMeta> file,
obsolete_files_.push_back( obsolete_files_.push_back(
std::make_pair(file->file_number(), obsolete_sequence)); std::make_pair(file->file_number(), obsolete_sequence));
file->FileStateTransit(BlobFileMeta::FileEvent::kDelete); file->FileStateTransit(BlobFileMeta::FileEvent::kDelete);
SubStats(stats_, cf_id_, file->GetDiscardableRatioLevel(), 1);
SubStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_SIZE, SubStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_SIZE,
file->live_data_size()); file->live_data_size());
SubStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_FILE_SIZE, SubStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_FILE_SIZE,
......
src/blob_storage.h
View file @ e6fa07d5
...@@ -73,8 +73,6 @@ class BlobStorage { ...@@ -73,8 +73,6 @@ class BlobStorage {
void InitializeAllFiles() { void InitializeAllFiles() {
for (auto& file : files_) { for (auto& file : files_) {
file.second->FileStateTransit(BlobFileMeta::FileEvent::kDbRestart); file.second->FileStateTransit(BlobFileMeta::FileEvent::kDbRestart);
auto level = file.second->GetDiscardableRatioLevel();
AddStats(stats_, cf_id_, level, 1);
} }
ComputeGCScore(); ComputeGCScore();
} }
......
src/db_impl.cc
View file @ e6fa07d5
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include <inttypes.h> #include <inttypes.h>
#include "logging/log_buffer.h" #include "logging/log_buffer.h"
#include "monitoring/statistics_impl.h"
#include "port/port.h" #include "port/port.h"
#include "util/autovector.h" #include "util/autovector.h"
#include "util/threadpool_imp.h" #include "util/threadpool_imp.h"
...@@ -57,10 +58,10 @@ class TitanDBImpl::FileManager : public BlobFileManager { ...@@ -57,10 +58,10 @@ class TitanDBImpl::FileManager : public BlobFileManager {
VersionEdit edit; VersionEdit edit;
edit.SetColumnFamilyID(cf_id); edit.SetColumnFamilyID(cf_id);
for (auto& file : files) { for (auto& file : files) {
RecordTick(statistics(db_->stats_.get()), BLOB_DB_BLOB_FILE_SYNCED); RecordTick(statistics(db_->stats_.get()), TITAN_BLOB_FILE_SYNCED);
{ {
StopWatch sync_sw(db_->env_, statistics(db_->stats_.get()), StopWatch sync_sw(db_->env_, statistics(db_->stats_.get()),
BLOB_DB_BLOB_FILE_SYNC_MICROS); TITAN_BLOB_FILE_SYNC_MICROS);
s = file.second->GetFile()->Sync(false); s = file.second->GetFile()->Sync(false);
} }
if (s.ok()) { if (s.ok()) {
...@@ -605,8 +606,8 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options, ...@@ -605,8 +606,8 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options,
nullptr /*read_callback*/, &is_blob_index); nullptr /*read_callback*/, &is_blob_index);
if (!s.ok() || !is_blob_index) return s; if (!s.ok() || !is_blob_index) return s;
StopWatch get_sw(env_, statistics(stats_.get()), BLOB_DB_GET_MICROS); StopWatch get_sw(env_, statistics(stats_.get()), TITAN_GET_MICROS);
RecordTick(statistics(stats_.get()), BLOB_DB_NUM_GET); RecordTick(statistics(stats_.get()), TITAN_NUM_GET);
BlobIndex index; BlobIndex index;
s = index.DecodeFrom(value); s = index.DecodeFrom(value);
...@@ -625,10 +626,10 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options, ...@@ -625,10 +626,10 @@ Status TitanDBImpl::GetImpl(const ReadOptions& options,
if (storage) { if (storage) {
StopWatch read_sw(env_, statistics(stats_.get()), StopWatch read_sw(env_, statistics(stats_.get()),
BLOB_DB_BLOB_FILE_READ_MICROS); TITAN_BLOB_FILE_READ_MICROS);
s = storage->Get(options, index, &record, &buffer); s = storage->Get(options, index, &record, &buffer);
RecordTick(statistics(stats_.get()), BLOB_DB_NUM_KEYS_READ); RecordTick(statistics(stats_.get()), TITAN_BLOB_FILE_NUM_KEYS_READ);
RecordTick(statistics(stats_.get()), BLOB_DB_BLOB_FILE_BYTES_READ, RecordTick(statistics(stats_.get()), TITAN_BLOB_FILE_BYTES_READ,
index.blob_handle.size); index.blob_handle.size);
} else { } else {
ROCKS_LOG_ERROR(db_options_.info_log, ROCKS_LOG_ERROR(db_options_.info_log,
......
src/db_impl_gc.cc
View file @ e6fa07d5
...@@ -161,7 +161,6 @@ void TitanDBImpl::BackgroundCallGC() { ...@@ -161,7 +161,6 @@ void TitanDBImpl::BackgroundCallGC() {
Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
uint32_t column_family_id) { uint32_t column_family_id) {
mutex_.AssertHeld(); mutex_.AssertHeld();
StopWatch gc_sw(env_, statistics(stats_.get()), BLOB_DB_GC_MICROS);
std::unique_ptr<BlobGC> blob_gc; std::unique_ptr<BlobGC> blob_gc;
bool gc_merge_rewrite = false; bool gc_merge_rewrite = false;
...@@ -196,9 +195,11 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -196,9 +195,11 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
// TODO(@DorianZheng) Make sure enough room for GC // TODO(@DorianZheng) Make sure enough room for GC
if (UNLIKELY(!blob_gc)) { if (UNLIKELY(!blob_gc)) {
RecordTick(statistics(stats_.get()), TITAN_GC_NO_NEED, 1);
// Nothing to do // Nothing to do
ROCKS_LOG_BUFFER(log_buffer, "Titan GC nothing to do"); ROCKS_LOG_BUFFER(log_buffer, "Titan GC nothing to do");
} else { } else {
StopWatch gc_sw(env_, statistics(stats_.get()), TITAN_GC_MICROS);
BlobGCJob blob_gc_job(blob_gc.get(), db_, &mutex_, db_options_, BlobGCJob blob_gc_job(blob_gc.get(), db_, &mutex_, db_options_,
gc_merge_rewrite, env_, env_options_, gc_merge_rewrite, env_, env_options_,
blob_manager_.get(), blob_file_set_.get(), log_buffer, blob_manager_.get(), blob_file_set_.get(), log_buffer,
...@@ -217,7 +218,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -217,7 +218,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
if (blob_gc->trigger_next() && if (blob_gc->trigger_next() &&
(bg_gc_scheduled_ - 1 + gc_queue_.size() < (bg_gc_scheduled_ - 1 + gc_queue_.size() <
2 * static_cast<uint32_t>(db_options_.max_background_gc))) { 2 * static_cast<uint32_t>(db_options_.max_background_gc))) {
RecordTick(statistics(stats_.get()), GC_TRIGGER_NEXT, 1); RecordTick(statistics(stats_.get()), TITAN_GC_TRIGGER_NEXT, 1);
// There is still data remained to be GCed // There is still data remained to be GCed
// and the queue is not overwhelmed // and the queue is not overwhelmed
// then put this cf to GC queue for next GC // then put this cf to GC queue for next GC
...@@ -226,11 +227,11 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -226,11 +227,11 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
} }
if (s.ok()) { if (s.ok()) {
RecordTick(statistics(stats_.get()), GC_SUCCESS, 1); RecordTick(statistics(stats_.get()), TITAN_GC_SUCCESS, 1);
// Done // Done
} else { } else {
SetBGError(s); SetBGError(s);
RecordTick(statistics(stats_.get()), GC_FAIL, 1); RecordTick(statistics(stats_.get()), TITAN_GC_FAILURE, 1);
ROCKS_LOG_WARN(db_options_.info_log, "Titan GC error: %s", ROCKS_LOG_WARN(db_options_.info_log, "Titan GC error: %s",
s.ToString().c_str()); s.ToString().c_str());
} }
......
src/db_iter.h
View file @ e6fa07d5
...@@ -32,6 +32,11 @@ class TitanDBIterator : public Iterator { ...@@ -32,6 +32,11 @@ class TitanDBIterator : public Iterator {
stats_(stats), stats_(stats),
info_log_(info_log) {} info_log_(info_log) {}
~TitanDBIterator() {
RecordInHistogram(statistics(stats_), TITAN_ITER_TOUCH_BLOB_FILE_COUNT,
files_.size());
}
bool Valid() const override { return iter_->Valid() && status_.ok(); } bool Valid() const override { return iter_->Valid() && status_.ok(); }
Status status() const override { Status status() const override {
...@@ -46,36 +51,36 @@ class TitanDBIterator : public Iterator { ...@@ -46,36 +51,36 @@ class TitanDBIterator : public Iterator {
void SeekToFirst() override { void SeekToFirst() override {
iter_->SeekToFirst(); iter_->SeekToFirst();
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS); StopWatch seek_sw(env_, statistics(stats_), TITAN_SEEK_MICROS);
GetBlobValue(true); GetBlobValue(true);
RecordTick(statistics(stats_), BLOB_DB_NUM_SEEK); RecordTick(statistics(stats_), TITAN_NUM_SEEK);
} }
} }
void SeekToLast() override { void SeekToLast() override {
iter_->SeekToLast(); iter_->SeekToLast();
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS); StopWatch seek_sw(env_, statistics(stats_), TITAN_SEEK_MICROS);
GetBlobValue(false); GetBlobValue(false);
RecordTick(statistics(stats_), BLOB_DB_NUM_SEEK); RecordTick(statistics(stats_), TITAN_NUM_SEEK);
} }
} }
void Seek(const Slice& target) override { void Seek(const Slice& target) override {
iter_->Seek(target); iter_->Seek(target);
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS); StopWatch seek_sw(env_, statistics(stats_), TITAN_SEEK_MICROS);
GetBlobValue(true); GetBlobValue(true);
RecordTick(statistics(stats_), BLOB_DB_NUM_SEEK); RecordTick(statistics(stats_), TITAN_NUM_SEEK);
} }
} }
void SeekForPrev(const Slice& target) override { void SeekForPrev(const Slice& target) override {
iter_->SeekForPrev(target); iter_->SeekForPrev(target);
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch seek_sw(env_, statistics(stats_), BLOB_DB_SEEK_MICROS); StopWatch seek_sw(env_, statistics(stats_), TITAN_SEEK_MICROS);
GetBlobValue(false); GetBlobValue(false);
RecordTick(statistics(stats_), BLOB_DB_NUM_SEEK); RecordTick(statistics(stats_), TITAN_NUM_SEEK);
} }
} }
...@@ -83,9 +88,9 @@ class TitanDBIterator : public Iterator { ...@@ -83,9 +88,9 @@ class TitanDBIterator : public Iterator {
assert(Valid()); assert(Valid());
iter_->Next(); iter_->Next();
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch next_sw(env_, statistics(stats_), BLOB_DB_NEXT_MICROS); StopWatch next_sw(env_, statistics(stats_), TITAN_NEXT_MICROS);
GetBlobValue(true); GetBlobValue(true);
RecordTick(statistics(stats_), BLOB_DB_NUM_NEXT); RecordTick(statistics(stats_), TITAN_NUM_NEXT);
} }
} }
...@@ -93,9 +98,9 @@ class TitanDBIterator : public Iterator { ...@@ -93,9 +98,9 @@ class TitanDBIterator : public Iterator {
assert(Valid()); assert(Valid());
iter_->Prev(); iter_->Prev();
if (ShouldGetBlobValue()) { if (ShouldGetBlobValue()) {
StopWatch prev_sw(env_, statistics(stats_), BLOB_DB_PREV_MICROS); StopWatch prev_sw(env_, statistics(stats_), TITAN_PREV_MICROS);
GetBlobValue(false); GetBlobValue(false);
RecordTick(statistics(stats_), BLOB_DB_NUM_PREV); RecordTick(statistics(stats_), TITAN_NUM_PREV);
} }
} }
......
src/table_builder.cc
View file @ e6fa07d5
...@@ -114,7 +114,7 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value, ...@@ -114,7 +114,7 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value,
std::string* index_value) { std::string* index_value) {
if (!ok()) return; if (!ok()) return;
StopWatch write_sw(db_options_.env, statistics(stats_), StopWatch write_sw(db_options_.env, statistics(stats_),
BLOB_DB_BLOB_FILE_WRITE_MICROS); TITAN_BLOB_FILE_WRITE_MICROS);
if (!blob_builder_) { if (!blob_builder_) {
status_ = blob_manager_->NewFile(&blob_handle_); status_ = blob_manager_->NewFile(&blob_handle_);
...@@ -126,9 +126,9 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value, ...@@ -126,9 +126,9 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value,
new BlobFileBuilder(db_options_, cf_options_, blob_handle_->GetFile())); new BlobFileBuilder(db_options_, cf_options_, blob_handle_->GetFile()));
} }
RecordTick(statistics(stats_), BLOB_DB_NUM_KEYS_WRITTEN); RecordTick(statistics(stats_), TITAN_BLOB_FILE_NUM_KEYS_WRITTEN);
RecordInHistogram(statistics(stats_), BLOB_DB_KEY_SIZE, key.size()); RecordInHistogram(statistics(stats_), TITAN_KEY_SIZE, key.size());
RecordInHistogram(statistics(stats_), BLOB_DB_VALUE_SIZE, value.size()); RecordInHistogram(statistics(stats_), TITAN_VALUE_SIZE, value.size());
AddStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_SIZE, value.size()); AddStats(stats_, cf_id_, TitanInternalStats::LIVE_BLOB_SIZE, value.size());
bytes_written_ += key.size() + value.size(); bytes_written_ += key.size() + value.size();
...@@ -138,7 +138,7 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value, ...@@ -138,7 +138,7 @@ void TitanTableBuilder::AddBlob(const Slice& key, const Slice& value,
record.value = value; record.value = value;
index.file_number = blob_handle_->GetNumber(); index.file_number = blob_handle_->GetNumber();
blob_builder_->Add(record, &index.blob_handle); blob_builder_->Add(record, &index.blob_handle);
RecordTick(statistics(stats_), BLOB_DB_BLOB_FILE_BYTES_WRITTEN, RecordTick(statistics(stats_), TITAN_BLOB_FILE_BYTES_WRITTEN,
index.blob_handle.size); index.blob_handle.size);
bytes_written_ += record.size(); bytes_written_ += record.size();
if (ok()) { if (ok()) {
......
src/titan_stats.h
View file @ e6fa07d5
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include "rocksdb/iostats_context.h" #include "rocksdb/iostats_context.h"
#include "rocksdb/statistics.h" #include "rocksdb/statistics.h"
#include "titan/options.h" #include "titan/options.h"
#include "titan/statistics.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -136,32 +137,6 @@ class TitanInternalStats { ...@@ -136,32 +137,6 @@ class TitanInternalStats {
internal_op_stats_; internal_op_stats_;
}; };
enum TickerType : uint32_t {
BLOB_CACHE_HIT = TICKER_ENUM_MAX,
BLOB_CACHE_MISS,
GC_NO_NEED,
GC_REMAIN,
GC_DISCARDABLE,
GC_SAMPLE,
GC_SMALL_FILE,
GC_FAIL,
GC_SUCCESS,
GC_TRIGGER_NEXT,
TITAN_TICKER_ENUM_MAX,
};
enum HistogramType : uint32_t {
TITAN_MANIFEST_FILE_SYNC_MICROS = HISTOGRAM_ENUM_MAX,
GC_INPUT_FILE_SIZE,
GC_OUTPUT_FILE_SIZE,
TITAN_HISTOGRAM_ENUM_MAX,
};
class TitanStats { class TitanStats {
public: public:
TitanStats(Statistics* stats) : stats_(stats) {} TitanStats(Statistics* stats) : stats_(stats) {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment