Skip to content

T
titan
Commit 77cc387b authored by yiwu-arbug's avatar yiwu-arbug
Browse files
Options

Fix rocksdb logging macro (#218) 

This change is to fix ASAN test failure in CI.

The rocksdb logging.h file provides the ROCKS_LOG_XXX logging macro, and it contain a logic to shorten the source file being print. It use the length of its file path (e.g. "/path/to/rocksdb/logging/logging.h")  to truncate other source file path. Since Titan source could be in different directory, Titan source file path could be truncated incorrectly, or even cause out-of-range access to the string buffer. Copying the file into titan repo and make a Titan version of the macros to fix the problem.

Also doing clang format.
Signed-off-by: 's avatarYi Wu <yiwu@pingcap.com>
Signed-off-by: 's avatarYi Wu <yiwu@pingcap.com>
parent fd580375
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 151 additions and 72 deletions
src/blob_file_set.cc
View file @ 77cc387b
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#include <inttypes.h> #include <inttypes.h>
#include "edit_collector.h" #include "edit_collector.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -83,7 +84,7 @@ Status BlobFileSet::Recover() { ...@@ -83,7 +84,7 @@ Status BlobFileSet::Recover() {
s = collector.GetNextFileNumber(&next_file_number); s = collector.GetNextFileNumber(&next_file_number);
if (!s.ok()) return s; if (!s.ok()) return s;
next_file_number_.store(next_file_number); next_file_number_.store(next_file_number);
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Next blob file number is %" PRIu64 ".", next_file_number); "Next blob file number is %" PRIu64 ".", next_file_number);
} }
...@@ -105,7 +106,7 @@ Status BlobFileSet::Recover() { ...@@ -105,7 +106,7 @@ Status BlobFileSet::Recover() {
files_str.append("(obsolete)"); files_str.append("(obsolete)");
} }
} }
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Blob files for CF %" PRIu32 " found: %s", bs.first, "Blob files for CF %" PRIu32 " found: %s", bs.first,
files_str.c_str()); files_str.c_str());
// delete obsoleted files at reopen // delete obsoleted files at reopen
...@@ -125,7 +126,7 @@ Status BlobFileSet::Recover() { ...@@ -125,7 +126,7 @@ Status BlobFileSet::Recover() {
if (file_type != FileType::kBlobFile && if (file_type != FileType::kBlobFile &&
file_type != FileType::kDescriptorFile) file_type != FileType::kDescriptorFile)
continue; continue;
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan recovery delete obsolete file %s.", f.c_str()); "Titan recovery delete obsolete file %s.", f.c_str());
env_->DeleteFile(dirname_ + "/" + f); env_->DeleteFile(dirname_ + "/" + f);
} }
...@@ -241,7 +242,7 @@ Status BlobFileSet::DropColumnFamilies( ...@@ -241,7 +242,7 @@ Status BlobFileSet::DropColumnFamilies(
edit.SetColumnFamilyID(it->first); edit.SetColumnFamilyID(it->first);
for (auto& file : it->second->files_) { for (auto& file : it->second->files_) {
if (!file.second->is_obsolete()) { if (!file.second->is_obsolete()) {
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan add obsolete file [%" PRIu64 "]", "Titan add obsolete file [%" PRIu64 "]",
file.second->file_number()); file.second->file_number());
edit.DeleteBlobFile(file.first, obsolete_sequence); edit.DeleteBlobFile(file.first, obsolete_sequence);
...@@ -250,7 +251,7 @@ Status BlobFileSet::DropColumnFamilies( ...@@ -250,7 +251,7 @@ Status BlobFileSet::DropColumnFamilies(
s = LogAndApply(edit); s = LogAndApply(edit);
if (!s.ok()) return s; if (!s.ok()) return s;
} else { } else {
ROCKS_LOG_ERROR(db_options_.info_log, "column %u not found for drop\n", TITAN_LOG_ERROR(db_options_.info_log, "column %u not found for drop\n",
cf_id); cf_id);
return Status::NotFound("invalid column family"); return Status::NotFound("invalid column family");
} }
...@@ -269,7 +270,7 @@ Status BlobFileSet::MaybeDestroyColumnFamily(uint32_t cf_id) { ...@@ -269,7 +270,7 @@ Status BlobFileSet::MaybeDestroyColumnFamily(uint32_t cf_id) {
} }
return Status::OK(); return Status::OK();
} }
ROCKS_LOG_ERROR(db_options_.info_log, "column %u not found for destroy\n", TITAN_LOG_ERROR(db_options_.info_log, "column %u not found for destroy\n",
cf_id); cf_id);
return Status::NotFound("invalid column family"); return Status::NotFound("invalid column family");
} }
...@@ -292,7 +293,7 @@ Status BlobFileSet::DeleteBlobFilesInRanges(uint32_t cf_id, ...@@ -292,7 +293,7 @@ Status BlobFileSet::DeleteBlobFilesInRanges(uint32_t cf_id,
s = LogAndApply(edit); s = LogAndApply(edit);
return s; return s;
} }
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"column %u not found for delete blob files in ranges\n", "column %u not found for delete blob files in ranges\n",
cf_id); cf_id);
return Status::NotFound("invalid column family"); return Status::NotFound("invalid column family");
......
src/blob_gc_job.cc
View file @ 77cc387b
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <memory> #include <memory>
#include "blob_file_size_collector.h" #include "blob_file_size_collector.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -133,7 +134,7 @@ Status BlobGCJob::Run() { ...@@ -133,7 +134,7 @@ Status BlobGCJob::Run() {
} }
tmp.append(std::to_string(f->file_number())); tmp.append(std::to_string(f->file_number()));
} }
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Titan GC candidates[%s]", TITAN_LOG_BUFFER(log_buffer_, "[%s] Titan GC candidates[%s]",
blob_gc_->column_family_handle()->GetName().c_str(), blob_gc_->column_family_handle()->GetName().c_str(),
tmp.c_str()); tmp.c_str());
return DoRunGC(); return DoRunGC();
...@@ -214,7 +215,7 @@ Status BlobGCJob::DoRunGC() { ...@@ -214,7 +215,7 @@ Status BlobGCJob::DoRunGC() {
if (!s.ok()) { if (!s.ok()) {
break; break;
} }
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan new GC output file %" PRIu64 ".", "Titan new GC output file %" PRIu64 ".",
blob_file_handle->GetNumber()); blob_file_handle->GetNumber());
blob_file_builder = std::unique_ptr<BlobFileBuilder>( blob_file_builder = std::unique_ptr<BlobFileBuilder>(
...@@ -378,13 +379,13 @@ Status BlobGCJob::Finish() { ...@@ -378,13 +379,13 @@ Status BlobGCJob::Finish() {
TEST_SYNC_POINT("BlobGCJob::Finish::BeforeRewriteValidKeyToLSM"); TEST_SYNC_POINT("BlobGCJob::Finish::BeforeRewriteValidKeyToLSM");
s = RewriteValidKeyToLSM(); s = RewriteValidKeyToLSM();
if (!s.ok()) { if (!s.ok()) {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"[%s] GC job failed to rewrite keys to LSM: %s", "[%s] GC job failed to rewrite keys to LSM: %s",
blob_gc_->column_family_handle()->GetName().c_str(), blob_gc_->column_family_handle()->GetName().c_str(),
s.ToString().c_str()); s.ToString().c_str());
} }
} else { } else {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"[%s] GC job failed to install output blob files: %s", "[%s] GC job failed to install output blob files: %s",
blob_gc_->column_family_handle()->GetName().c_str(), blob_gc_->column_family_handle()->GetName().c_str(),
s.ToString().c_str()); s.ToString().c_str());
...@@ -433,7 +434,7 @@ Status BlobGCJob::InstallOutputBlobFiles() { ...@@ -433,7 +434,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
files.emplace_back(std::make_pair(file, std::move(builder.first))); files.emplace_back(std::make_pair(file, std::move(builder.first)));
} }
if (s.ok()) { if (s.ok()) {
ROCKS_LOG_BUFFER(log_buffer_, "[%s] output[%s]", TITAN_LOG_BUFFER(log_buffer_, "[%s] output[%s]",
blob_gc_->column_family_handle()->GetName().c_str(), blob_gc_->column_family_handle()->GetName().c_str(),
tmp.c_str()); tmp.c_str());
s = blob_file_manager_->BatchFinishFiles( s = blob_file_manager_->BatchFinishFiles(
...@@ -453,7 +454,7 @@ Status BlobGCJob::InstallOutputBlobFiles() { ...@@ -453,7 +454,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
to_delete_files.append(std::to_string(builder.first->GetNumber())); to_delete_files.append(std::to_string(builder.first->GetNumber()));
handles.emplace_back(std::move(builder.first)); handles.emplace_back(std::move(builder.first));
} }
ROCKS_LOG_BUFFER(log_buffer_, TITAN_LOG_BUFFER(log_buffer_,
"[%s] InstallOutputBlobFiles failed. Delete GC output " "[%s] InstallOutputBlobFiles failed. Delete GC output "
"files: %s", "files: %s",
blob_gc_->column_family_handle()->GetName().c_str(), blob_gc_->column_family_handle()->GetName().c_str(),
...@@ -463,7 +464,7 @@ Status BlobGCJob::InstallOutputBlobFiles() { ...@@ -463,7 +464,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
// LSM by mistake. // LSM by mistake.
Status status = blob_file_manager_->BatchDeleteFiles(handles); Status status = blob_file_manager_->BatchDeleteFiles(handles);
if (!status.ok()) { if (!status.ok()) {
ROCKS_LOG_WARN(db_options_.info_log, TITAN_LOG_WARN(db_options_.info_log,
"Delete GC output files[%s] failed: %s", "Delete GC output files[%s] failed: %s",
to_delete_files.c_str(), status.ToString().c_str()); to_delete_files.c_str(), status.ToString().c_str());
} }
...@@ -554,7 +555,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() { ...@@ -554,7 +555,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
if (blob_storage) { if (blob_storage) {
auto file = blob_storage->FindFile(blob_file.first).lock(); auto file = blob_storage->FindFile(blob_file.first).lock();
if (!file) { if (!file) {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"Blob File %" PRIu64 " not found when GC.", "Blob File %" PRIu64 " not found when GC.",
blob_file.first); blob_file.first);
continue; continue;
...@@ -565,7 +566,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() { ...@@ -565,7 +566,7 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
blob_storage->ComputeGCScore(); blob_storage->ComputeGCScore();
} else { } else {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"Column family id:%" PRIu32 " not Found when GC.", cf_id); "Column family id:%" PRIu32 " not Found when GC.", cf_id);
} }
} }
...@@ -586,7 +587,7 @@ Status BlobGCJob::DeleteInputBlobFiles() { ...@@ -586,7 +587,7 @@ Status BlobGCJob::DeleteInputBlobFiles() {
VersionEdit edit; VersionEdit edit;
edit.SetColumnFamilyID(blob_gc_->column_family_handle()->GetID()); edit.SetColumnFamilyID(blob_gc_->column_family_handle()->GetID());
for (const auto& file : blob_gc_->inputs()) { for (const auto& file : blob_gc_->inputs()) {
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan add obsolete file [%" PRIu64 "] range [%s, %s]", "Titan add obsolete file [%" PRIu64 "] range [%s, %s]",
file->file_number(), file->file_number(),
Slice(file->smallest_key()).ToString(true).c_str(), Slice(file->smallest_key()).ToString(true).c_str(),
......
src/blob_gc_picker.cc
View file @ 77cc387b
...@@ -6,6 +6,8 @@ ...@@ -6,6 +6,8 @@
#include <inttypes.h> #include <inttypes.h>
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -34,7 +36,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC( ...@@ -34,7 +36,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
if (!CheckBlobFile(blob_file.get())) { if (!CheckBlobFile(blob_file.get())) {
// Skip this file id this file is being GCed // Skip this file id this file is being GCed
// or this file had been GCed // or this file had been GCed
ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 " no need gc", TITAN_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 " no need gc",
blob_file->file_number()); blob_file->file_number());
continue; continue;
} }
...@@ -53,7 +55,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC( ...@@ -53,7 +55,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
if (next_gc_size > cf_options_.min_gc_batch_size) { if (next_gc_size > cf_options_.min_gc_batch_size) {
maybe_continue_next_time = true; maybe_continue_next_time = true;
RecordTick(statistics(stats_), TITAN_GC_REMAIN, 1); RecordTick(statistics(stats_), TITAN_GC_REMAIN, 1);
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"remain more than %" PRIu64 "remain more than %" PRIu64
" bytes to be gc and trigger after this gc", " bytes to be gc and trigger after this gc",
next_gc_size); next_gc_size);
...@@ -61,7 +63,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC( ...@@ -61,7 +63,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
} }
} }
} }
ROCKS_LOG_DEBUG(db_options_.info_log, TITAN_LOG_DEBUG(db_options_.info_log,
"got batch size %" PRIu64 ", estimate output %" PRIu64 "got batch size %" PRIu64 ", estimate output %" PRIu64
" bytes", " bytes",
batch_size, estimate_output_size); batch_size, estimate_output_size);
......
src/blob_storage.cc
View file @ 77cc387b
#include "blob_storage.h" #include "blob_storage.h"
#include "blob_file_set.h" #include "blob_file_set.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -58,7 +60,7 @@ Status BlobStorage::GetBlobFilesInRanges(const RangePtr* ranges, size_t n, ...@@ -58,7 +60,7 @@ Status BlobStorage::GetBlobFilesInRanges(const RangePtr* ranges, size_t n,
assert(it->second->smallest_key().empty() || assert(it->second->smallest_key().empty() ||
(!begin || cmp->Compare(it->second->smallest_key(), *begin) >= 0)); (!begin || cmp->Compare(it->second->smallest_key(), *begin) >= 0));
} }
ROCKS_LOG_INFO( TITAN_LOG_INFO(
db_options_.info_log, db_options_.info_log,
"Get %" PRIuPTR " blob files [%s] in the range [%s, %s%c", "Get %" PRIuPTR " blob files [%s] in the range [%s, %s%c",
files->size(), tmp.c_str(), begin ? begin->ToString(true).c_str() : " ", files->size(), tmp.c_str(), begin ? begin->ToString(true).c_str() : " ",
...@@ -171,7 +173,7 @@ void BlobStorage::GetObsoleteFiles(std::vector<std::string>* obsolete_files, ...@@ -171,7 +173,7 @@ void BlobStorage::GetObsoleteFiles(std::vector<std::string>* obsolete_files,
// remove obsolete files // remove obsolete files
bool __attribute__((__unused__)) removed = RemoveFile(file_number); bool __attribute__((__unused__)) removed = RemoveFile(file_number);
assert(removed); assert(removed);
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Obsolete blob file %" PRIu64 " (obsolete at %" PRIu64 "Obsolete blob file %" PRIu64 " (obsolete at %" PRIu64
") not visible to oldest snapshot %" PRIu64 ", delete it.", ") not visible to oldest snapshot %" PRIu64 ", delete it.",
file_number, obsolete_sequence, oldest_sequence); file_number, obsolete_sequence, oldest_sequence);
......
src/compaction_filter.h
View file @ 77cc387b
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include "db_impl.h" #include "db_impl.h"
#include "rocksdb/compaction_filter.h" #include "rocksdb/compaction_filter.h"
#include "titan_logging.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
namespace rocksdb { namespace rocksdb {
...@@ -48,7 +49,7 @@ class TitanCompactionFilter final : public CompactionFilter { ...@@ -48,7 +49,7 @@ class TitanCompactionFilter final : public CompactionFilter {
Slice original_value(value.data()); Slice original_value(value.data());
Status s = blob_index.DecodeFrom(&original_value); Status s = blob_index.DecodeFrom(&original_value);
if (!s.ok()) { if (!s.ok()) {
ROCKS_LOG_ERROR(db_->db_options_.info_log, TITAN_LOG_ERROR(db_->db_options_.info_log,
"[%s] Unable to decode blob index", cf_name_.c_str()); "[%s] Unable to decode blob index", cf_name_.c_str());
// TODO(yiwu): Better to fail the compaction as well, but current // TODO(yiwu): Better to fail the compaction as well, but current
// compaction filter API doesn't support it. // compaction filter API doesn't support it.
......
src/db_impl.cc
View file @ 77cc387b
This diff is collapsed.
src/db_impl_files.cc
View file @ 77cc387b
#include "db_impl.h" #include "db_impl.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -20,12 +21,12 @@ Status TitanDBImpl::PurgeObsoleteFilesImpl() { ...@@ -20,12 +21,12 @@ Status TitanDBImpl::PurgeObsoleteFilesImpl() {
candidate_files.end()); candidate_files.end());
for (const auto& candidate_file : candidate_files) { for (const auto& candidate_file : candidate_files) {
ROCKS_LOG_INFO(db_options_.info_log, "Titan deleting obsolete file [%s]", TITAN_LOG_INFO(db_options_.info_log, "Titan deleting obsolete file [%s]",
candidate_file.c_str()); candidate_file.c_str());
Status delete_status = env_->DeleteFile(candidate_file); Status delete_status = env_->DeleteFile(candidate_file);
if (!s.ok()) { if (!s.ok()) {
// Move on despite error deleting the file. // Move on despite error deleting the file.
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"Titan deleting file [%s] failed, status:%s", "Titan deleting file [%s] failed, status:%s",
candidate_file.c_str(), s.ToString().c_str()); candidate_file.c_str(), s.ToString().c_str());
s = delete_status; s = delete_status;
......
src/db_impl_gc.cc
View file @ 77cc387b
#include "db_impl.h"
#include "test_util/sync_point.h"
#include "blob_file_iterator.h" #include "blob_file_iterator.h"
#include "blob_file_size_collector.h" #include "blob_file_size_collector.h"
#include "blob_gc_job.h" #include "blob_gc_job.h"
#include "blob_gc_picker.h" #include "blob_gc_picker.h"
#include "db_impl.h"
#include "test_util/sync_point.h"
#include "titan_logging.h"
#include "util.h" #include "util.h"
namespace rocksdb { namespace rocksdb {
...@@ -175,7 +174,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -175,7 +174,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
blob_storage = blob_file_set_->GetBlobStorage(column_family_id).lock(); blob_storage = blob_file_set_->GetBlobStorage(column_family_id).lock();
} else { } else {
TEST_SYNC_POINT_CALLBACK("TitanDBImpl::BackgroundGC:CFDropped", nullptr); TEST_SYNC_POINT_CALLBACK("TitanDBImpl::BackgroundGC:CFDropped", nullptr);
ROCKS_LOG_BUFFER(log_buffer, "GC skip dropped colum family [%s].", TITAN_LOG_BUFFER(log_buffer, "GC skip dropped colum family [%s].",
cf_info_[column_family_id].name.c_str()); cf_info_[column_family_id].name.c_str());
} }
if (blob_storage != nullptr) { if (blob_storage != nullptr) {
...@@ -199,7 +198,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -199,7 +198,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
if (UNLIKELY(!blob_gc)) { if (UNLIKELY(!blob_gc)) {
RecordTick(statistics(stats_.get()), TITAN_GC_NO_NEED, 1); RecordTick(statistics(stats_.get()), TITAN_GC_NO_NEED, 1);
// Nothing to do // Nothing to do
ROCKS_LOG_BUFFER(log_buffer, "Titan GC nothing to do"); TITAN_LOG_BUFFER(log_buffer, "Titan GC nothing to do");
} else { } else {
StopWatch gc_sw(env_, statistics(stats_.get()), TITAN_GC_MICROS); StopWatch gc_sw(env_, statistics(stats_.get()), TITAN_GC_MICROS);
BlobGCJob blob_gc_job(blob_gc.get(), db_, &mutex_, db_options_, BlobGCJob blob_gc_job(blob_gc.get(), db_, &mutex_, db_options_,
...@@ -236,7 +235,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer, ...@@ -236,7 +235,7 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
} else { } else {
SetBGError(s); SetBGError(s);
RecordTick(statistics(stats_.get()), TITAN_GC_FAILURE, 1); RecordTick(statistics(stats_.get()), TITAN_GC_FAILURE, 1);
ROCKS_LOG_WARN(db_options_.info_log, "Titan GC error: %s", TITAN_LOG_WARN(db_options_.info_log, "Titan GC error: %s",
s.ToString().c_str()); s.ToString().c_str());
} }
......
src/db_iter.h
View file @ 77cc387b
...@@ -10,9 +10,8 @@ ...@@ -10,9 +10,8 @@
#include <unordered_map> #include <unordered_map>
#include "db/db_iter.h" #include "db/db_iter.h"
#include "logging/logging.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "titan_logging.h"
#include "titan_stats.h" #include "titan_stats.h"
namespace rocksdb { namespace rocksdb {
...@@ -135,7 +134,7 @@ class TitanDBIterator : public Iterator { ...@@ -135,7 +134,7 @@ class TitanDBIterator : public Iterator {
BlobIndex index; BlobIndex index;
status_ = DecodeInto(iter_->value(), &index); status_ = DecodeInto(iter_->value(), &index);
if (!status_.ok()) { if (!status_.ok()) {
ROCKS_LOG_ERROR(info_log_, TITAN_LOG_ERROR(info_log_,
"Titan iterator: failed to decode blob index %s: %s", "Titan iterator: failed to decode blob index %s: %s",
iter_->value().ToString(true /*hex*/).c_str(), iter_->value().ToString(true /*hex*/).c_str(),
status_.ToString().c_str()); status_.ToString().c_str());
...@@ -153,7 +152,7 @@ class TitanDBIterator : public Iterator { ...@@ -153,7 +152,7 @@ class TitanDBIterator : public Iterator {
} else { } else {
status_ = DecodeInto(iter_->value(), &index); status_ = DecodeInto(iter_->value(), &index);
if (!status_.ok()) { if (!status_.ok()) {
ROCKS_LOG_ERROR(info_log_, TITAN_LOG_ERROR(info_log_,
"Titan iterator: failed to decode blob index %s: %s", "Titan iterator: failed to decode blob index %s: %s",
iter_->value().ToString(true /*hex*/).c_str(), iter_->value().ToString(true /*hex*/).c_str(),
status_.ToString().c_str()); status_.ToString().c_str());
...@@ -170,7 +169,7 @@ class TitanDBIterator : public Iterator { ...@@ -170,7 +169,7 @@ class TitanDBIterator : public Iterator {
std::unique_ptr<BlobFilePrefetcher> prefetcher; std::unique_ptr<BlobFilePrefetcher> prefetcher;
status_ = storage_->NewPrefetcher(index.file_number, &prefetcher); status_ = storage_->NewPrefetcher(index.file_number, &prefetcher);
if (!status_.ok()) { if (!status_.ok()) {
ROCKS_LOG_ERROR( TITAN_LOG_ERROR(
info_log_, info_log_,
"Titan iterator: failed to create prefetcher for blob file %" PRIu64 "Titan iterator: failed to create prefetcher for blob file %" PRIu64
": %s", ": %s",
...@@ -183,7 +182,7 @@ class TitanDBIterator : public Iterator { ...@@ -183,7 +182,7 @@ class TitanDBIterator : public Iterator {
buffer_.Reset(); buffer_.Reset();
status_ = it->second->Get(options_, index.blob_handle, &record_, &buffer_); status_ = it->second->Get(options_, index.blob_handle, &record_, &buffer_);
if (!status_.ok()) { if (!status_.ok()) {
ROCKS_LOG_ERROR( TITAN_LOG_ERROR(
info_log_, info_log_,
"Titan iterator: failed to read blob value from file %" PRIu64 "Titan iterator: failed to read blob value from file %" PRIu64
", offset %" PRIu64 ", size %" PRIu64 ": %s\n", ", offset %" PRIu64 ", size %" PRIu64 ": %s\n",
......
src/edit_collector.h
View file @ 77cc387b
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include <unordered_map> #include <unordered_map>
#include "blob_file_set.h" #include "blob_file_set.h"
#include "titan_logging.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "version_edit.h" #include "version_edit.h"
...@@ -144,13 +145,13 @@ class EditCollector { ...@@ -144,13 +145,13 @@ class EditCollector {
auto blob = storage->FindFile(number).lock(); auto blob = storage->FindFile(number).lock();
if (blob) { if (blob) {
if (blob->is_obsolete()) { if (blob->is_obsolete()) {
ROCKS_LOG_ERROR(storage->db_options().info_log, TITAN_LOG_ERROR(storage->db_options().info_log,
"blob file %" PRIu64 " has been deleted before\n", "blob file %" PRIu64 " has been deleted before\n",
number); number);
return Status::Corruption("Blob file " + ToString(number) + return Status::Corruption("Blob file " + ToString(number) +
" has been deleted before"); " has been deleted before");
} else { } else {
ROCKS_LOG_ERROR(storage->db_options().info_log, TITAN_LOG_ERROR(storage->db_options().info_log,
"blob file %" PRIu64 " has been added before\n", "blob file %" PRIu64 " has been added before\n",
number); number);
return Status::Corruption("Blob file " + ToString(number) + return Status::Corruption("Blob file " + ToString(number) +
...@@ -166,13 +167,13 @@ class EditCollector { ...@@ -166,13 +167,13 @@ class EditCollector {
} }
auto blob = storage->FindFile(number).lock(); auto blob = storage->FindFile(number).lock();
if (!blob) { if (!blob) {
ROCKS_LOG_ERROR(storage->db_options().info_log, TITAN_LOG_ERROR(storage->db_options().info_log,
"blob file %" PRIu64 " doesn't exist before\n", "blob file %" PRIu64 " doesn't exist before\n",
number); number);
return Status::Corruption("Blob file " + ToString(number) + return Status::Corruption("Blob file " + ToString(number) +
" doesn't exist before"); " doesn't exist before");
} else if (blob->is_obsolete()) { } else if (blob->is_obsolete()) {
ROCKS_LOG_ERROR(storage->db_options().info_log, TITAN_LOG_ERROR(storage->db_options().info_log,
"blob file %" PRIu64 " has been deleted already\n", "blob file %" PRIu64 " has been deleted already\n",
number); number);
return Status::Corruption("Blob file " + ToString(number) + return Status::Corruption("Blob file " + ToString(number) +
......
src/options.cc
View file @ 77cc387b
...@@ -6,25 +6,25 @@ ...@@ -6,25 +6,25 @@
#include <inttypes.h> #include <inttypes.h>
#include "logging/logging.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
void TitanDBOptions::Dump(Logger* logger) const { void TitanDBOptions::Dump(Logger* logger) const {
ROCKS_LOG_HEADER(logger, "TitanDBOptions.dirname : %s", TITAN_LOG_HEADER(logger, "TitanDBOptions.dirname : %s",
dirname.c_str()); dirname.c_str());
ROCKS_LOG_HEADER(logger, "TitanDBOptions.disable_background_gc : %d", TITAN_LOG_HEADER(logger, "TitanDBOptions.disable_background_gc : %d",
static_cast<int>(disable_background_gc)); static_cast<int>(disable_background_gc));
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanDBOptions.max_background_gc : %" PRIi32, "TitanDBOptions.max_background_gc : %" PRIi32,
max_background_gc); max_background_gc);
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanDBOptions.purge_obsolete_files_period_sec: %" PRIu32, "TitanDBOptions.purge_obsolete_files_period_sec: %" PRIu32,
purge_obsolete_files_period_sec); purge_obsolete_files_period_sec);
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanDBOptions.titan_stats_dump_period_sec: %" PRIu32, "TitanDBOptions.titan_stats_dump_period_sec: %" PRIu32,
titan_stats_dump_period_sec); titan_stats_dump_period_sec);
} }
...@@ -48,7 +48,7 @@ TitanCFOptions::TitanCFOptions(const ColumnFamilyOptions& cf_opts, ...@@ -48,7 +48,7 @@ TitanCFOptions::TitanCFOptions(const ColumnFamilyOptions& cf_opts,
immutable_opts.skip_value_in_compaction_filter) {} immutable_opts.skip_value_in_compaction_filter) {}
void TitanCFOptions::Dump(Logger* logger) const { void TitanCFOptions::Dump(Logger* logger) const {
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanCFOptions.min_blob_size : %" PRIu64, "TitanCFOptions.min_blob_size : %" PRIu64,
min_blob_size); min_blob_size);
std::string compression_str = "unknown"; std::string compression_str = "unknown";
...@@ -58,45 +58,45 @@ void TitanCFOptions::Dump(Logger* logger) const { ...@@ -58,45 +58,45 @@ void TitanCFOptions::Dump(Logger* logger) const {
break; break;
} }
} }
ROCKS_LOG_HEADER(logger, "TitanCFOptions.blob_file_compression : %s", TITAN_LOG_HEADER(logger, "TitanCFOptions.blob_file_compression : %s",
compression_str.c_str()); compression_str.c_str());
ROCKS_LOG_HEADER(logger, "TItanCFOptions.blob_file_compression_options: "); TITAN_LOG_HEADER(logger, "TItanCFOptions.blob_file_compression_options: ");
ROCKS_LOG_HEADER(logger, " window_bits : %d", TITAN_LOG_HEADER(logger, " window_bits : %d",
blob_file_compression_options.window_bits); blob_file_compression_options.window_bits);
ROCKS_LOG_HEADER(logger, " level : %d", TITAN_LOG_HEADER(logger, " level : %d",
blob_file_compression_options.level); blob_file_compression_options.level);
ROCKS_LOG_HEADER(logger, " strategy : %d", TITAN_LOG_HEADER(logger, " strategy : %d",
blob_file_compression_options.strategy); blob_file_compression_options.strategy);
ROCKS_LOG_HEADER(logger, " max_dict_bytes : %" PRIu32, TITAN_LOG_HEADER(logger, " max_dict_bytes : %" PRIu32,
blob_file_compression_options.max_dict_bytes); blob_file_compression_options.max_dict_bytes);
ROCKS_LOG_HEADER(logger, " zstd_max_train_bytes : %" PRIu32, TITAN_LOG_HEADER(logger, " zstd_max_train_bytes : %" PRIu32,
blob_file_compression_options.zstd_max_train_bytes); blob_file_compression_options.zstd_max_train_bytes);
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanCFOptions.blob_file_target_size : %" PRIu64, "TitanCFOptions.blob_file_target_size : %" PRIu64,
blob_file_target_size); blob_file_target_size);
ROCKS_LOG_HEADER(logger, "TitanCFOptions.blob_cache : %p", TITAN_LOG_HEADER(logger, "TitanCFOptions.blob_cache : %p",
blob_cache.get()); blob_cache.get());
if (blob_cache != nullptr) { if (blob_cache != nullptr) {
ROCKS_LOG_HEADER(logger, "%s", blob_cache->GetPrintableOptions().c_str()); TITAN_LOG_HEADER(logger, "%s", blob_cache->GetPrintableOptions().c_str());
} }
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanCFOptions.max_gc_batch_size : %" PRIu64, "TitanCFOptions.max_gc_batch_size : %" PRIu64,
max_gc_batch_size); max_gc_batch_size);
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanCFOptions.min_gc_batch_size : %" PRIu64, "TitanCFOptions.min_gc_batch_size : %" PRIu64,
min_gc_batch_size); min_gc_batch_size);
ROCKS_LOG_HEADER(logger, "TitanCFOptions.blob_file_discardable_ratio : %lf", TITAN_LOG_HEADER(logger, "TitanCFOptions.blob_file_discardable_ratio : %lf",
blob_file_discardable_ratio); blob_file_discardable_ratio);
ROCKS_LOG_HEADER(logger, "TitanCFOptions.sample_file_size_ratio : %lf", TITAN_LOG_HEADER(logger, "TitanCFOptions.sample_file_size_ratio : %lf",
sample_file_size_ratio); sample_file_size_ratio);
ROCKS_LOG_HEADER(logger, TITAN_LOG_HEADER(logger,
"TitanCFOptions.merge_small_file_threshold : %" PRIu64, "TitanCFOptions.merge_small_file_threshold : %" PRIu64,
merge_small_file_threshold); merge_small_file_threshold);
std::string blob_run_mode_str = "unknown"; std::string blob_run_mode_str = "unknown";
if (blob_run_mode_to_string.count(blob_run_mode) > 0) { if (blob_run_mode_to_string.count(blob_run_mode) > 0) {
blob_run_mode_str = blob_run_mode_to_string.at(blob_run_mode); blob_run_mode_str = blob_run_mode_to_string.at(blob_run_mode);
} }
ROCKS_LOG_HEADER(logger, "TitanCFOptions.blob_run_mode : %s", TITAN_LOG_HEADER(logger, "TitanCFOptions.blob_run_mode : %s",
blob_run_mode_str.c_str()); blob_run_mode_str.c_str());
} }
......
src/table_builder.cc
View file @ 77cc387b
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#include <inttypes.h> #include <inttypes.h>
#include "monitoring/statistics.h" #include "monitoring/statistics.h"
#include "titan_logging.h"
namespace rocksdb { namespace rocksdb {
namespace titandb { namespace titandb {
...@@ -109,7 +110,7 @@ void TitanTableBuilder::Add(const Slice& key, const Slice& value) { ...@@ -109,7 +110,7 @@ void TitanTableBuilder::Add(const Slice& key, const Slice& value) {
if (ok()) return; if (ok()) return;
} else { } else {
++error_read_cnt_; ++error_read_cnt_;
ROCKS_LOG_DEBUG(db_options_.info_log, TITAN_LOG_DEBUG(db_options_.info_log,
"Read file %" PRIu64 " error during level merge: %s", "Read file %" PRIu64 " error during level merge: %s",
index.file_number, get_status.ToString().c_str()); index.file_number, get_status.ToString().c_str());
} }
...@@ -148,7 +149,7 @@ void TitanTableBuilder::AddBlob(const ParsedInternalKey& ikey, ...@@ -148,7 +149,7 @@ void TitanTableBuilder::AddBlob(const ParsedInternalKey& ikey,
if (!blob_builder_) { if (!blob_builder_) {
status_ = blob_manager_->NewFile(&blob_handle_); status_ = blob_manager_->NewFile(&blob_handle_);
if (!ok()) return; if (!ok()) return;
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan table builder created new blob file %" PRIu64 ".", "Titan table builder created new blob file %" PRIu64 ".",
blob_handle_->GetNumber()); blob_handle_->GetNumber());
blob_builder_.reset( blob_builder_.reset(
...@@ -225,7 +226,7 @@ void TitanTableBuilder::FinishBlobFile() { ...@@ -225,7 +226,7 @@ void TitanTableBuilder::FinishBlobFile() {
AddToBaseTable(contexts); AddToBaseTable(contexts);
if (s.ok() && ok()) { if (s.ok() && ok()) {
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan table builder finish output file %" PRIu64 ".", "Titan table builder finish output file %" PRIu64 ".",
blob_handle_->GetNumber()); blob_handle_->GetNumber());
std::shared_ptr<BlobFileMeta> file = std::make_shared<BlobFileMeta>( std::shared_ptr<BlobFileMeta> file = std::make_shared<BlobFileMeta>(
...@@ -236,7 +237,7 @@ void TitanTableBuilder::FinishBlobFile() { ...@@ -236,7 +237,7 @@ void TitanTableBuilder::FinishBlobFile() {
finished_blobs_.push_back({file, std::move(blob_handle_)}); finished_blobs_.push_back({file, std::move(blob_handle_)});
blob_builder_.reset(); blob_builder_.reset();
} else { } else {
ROCKS_LOG_WARN( TITAN_LOG_WARN(
db_options_.info_log, db_options_.info_log,
"Titan table builder finish failed. Delete output file %" PRIu64 ".", "Titan table builder finish failed. Delete output file %" PRIu64 ".",
blob_handle_->GetNumber()); blob_handle_->GetNumber());
...@@ -264,13 +265,13 @@ Status TitanTableBuilder::Finish() { ...@@ -264,13 +265,13 @@ Status TitanTableBuilder::Finish() {
base_builder_->Finish(); base_builder_->Finish();
status_ = blob_manager_->BatchFinishFiles(cf_id_, finished_blobs_); status_ = blob_manager_->BatchFinishFiles(cf_id_, finished_blobs_);
if (!status_.ok()) { if (!status_.ok()) {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"Titan table builder failed on finish: %s", "Titan table builder failed on finish: %s",
status_.ToString().c_str()); status_.ToString().c_str());
} }
UpdateInternalOpStats(); UpdateInternalOpStats();
if (error_read_cnt_ > 0) { if (error_read_cnt_ > 0) {
ROCKS_LOG_ERROR(db_options_.info_log, TITAN_LOG_ERROR(db_options_.info_log,
"Read file error %" PRIu64 " times during level merge", "Read file error %" PRIu64 " times during level merge",
error_read_cnt_); error_read_cnt_);
} }
...@@ -280,7 +281,7 @@ Status TitanTableBuilder::Finish() { ...@@ -280,7 +281,7 @@ Status TitanTableBuilder::Finish() {
void TitanTableBuilder::Abandon() { void TitanTableBuilder::Abandon() {
base_builder_->Abandon(); base_builder_->Abandon();
if (blob_builder_) { if (blob_builder_) {
ROCKS_LOG_INFO(db_options_.info_log, TITAN_LOG_INFO(db_options_.info_log,
"Titan table builder abandoned. Delete output file %" PRIu64 "Titan table builder abandoned. Delete output file %" PRIu64
".", ".",
blob_handle_->GetNumber()); blob_handle_->GetNumber());
......
src/titan_logging.h 0 → 100644
View file @ 77cc387b
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// Must not be included from any .h files to avoid polluting the namespace
// with macros.
#pragma once
// This file is adapted from rocksdb/logging/logging.h.
// We make a copy of this file because we want to have source files under
// Titan directory being shorten.
// Helper macros that include information about file name and line number
#define TITAN_LOG_STRINGIFY(x) #x
#define TITAN_LOG_TOSTRING(x) TITAN_LOG_STRINGIFY(x)
#define TITAN_LOG_PREPEND_FILE_LINE(FMT) \
("[%s:" TITAN_LOG_TOSTRING(__LINE__) "] " FMT)
inline const char* TitanLogShorterFileName(const char* file) {
// 15 is the length of "titan_logging.h".
// If the name of this file changed, please change this number, too.
return file + (sizeof(__FILE__) > 15 ? sizeof(__FILE__) - 15 : 0);
}
// Don't inclide file/line info in HEADER level
#define TITAN_LOG_HEADER(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::HEADER_LEVEL, LGR, FMT, ##__VA_ARGS__)
#define TITAN_LOG_DEBUG(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::DEBUG_LEVEL, LGR, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_INFO(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::INFO_LEVEL, LGR, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_WARN(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::WARN_LEVEL, LGR, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_ERROR(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::ERROR_LEVEL, LGR, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_FATAL(LGR, FMT, ...) \
rocksdb::Log(InfoLogLevel::FATAL_LEVEL, LGR, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_BUFFER(LOG_BUF, FMT, ...) \
rocksdb::LogToBuffer(LOG_BUF, TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_BUFFER_MAX_SZ(LOG_BUF, MAX_LOG_SIZE, FMT, ...) \
rocksdb::LogToBuffer(LOG_BUF, MAX_LOG_SIZE, \
TITAN_LOG_PREPEND_FILE_LINE(FMT), \
TitanLogShorterFileName(__FILE__), ##__VA_ARGS__)
#define TITAN_LOG_DETAILS(LGR, FMT, ...) \
; // due to overhead by default skip such lines
// TITAN_LOG_DEBUG(LGR, FMT, ##__VA_ARGS__)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment