Add manifest_dump tool (#204)

Adding a tool to dump Titan manifest file, for easy debugging.

Sample output:
```
next_file_number: 93015
column family: 1

column family: 2

column family: 3

column family: 0
file 92978, size 44644407, level 0
file 92997, size 44513780, level 0
file 93001, size 44639151, level 0
file 93005, size 44420372, level 0
file 93009, size 44595346, level 0
file 93010, size 23167116, level 0
file 93011, size 38966307, level 0
file 93012, size 21745288, level 0
file 93014, size 44181211, level 0
```
Signed-off-by: Yi Wu <yiwu@pingcap.com>

CMakeLists.txt

@@ -144,6 +144,10 @@ if (WITH_TITAN_TOOLS)
   add_executable(titandb_bench tools/db_bench.cc tools/db_bench_tool.cc)
   target_include_directories(titandb_bench PRIVATE ${gflags_INCLUDE_DIR})
   target_link_libraries(titandb_bench ${TOOLS_LIBS})
+
+  add_executable(titan_manifest_dump tools/manifest_dump.cc)
+  target_include_directories(titan_manifest_dump PRIVATE ${gflags_INCLUDE_DIR})
+  target_link_libraries(titan_manifest_dump ${TOOLS_LIBS})
 endif()
 
 # Installation - copy lib/ and include/

src/blob_file_set.cc

@@ -38,13 +38,6 @@ Status BlobFileSet::Open(
 }
 
 Status BlobFileSet::Recover() {
-  struct LogReporter : public log::Reader::Reporter {
-    Status* status;
-    void Corruption(size_t, const Status& s) override {
-      if (status->ok()) *status = s;
-    }
-  };
-
   // Reads "CURRENT" file, which contains the name of the current manifest file.
   std::string manifest;
   Status s = ReadFileToString(env_, CurrentFileName(dirname_), &manifest);

src/blob_file_set.h

@@ -20,6 +20,13 @@
 namespace rocksdb {
 namespace titandb {
 
+struct LogReporter : public log::Reader::Reporter {
+  Status* status;
+  void Corruption(size_t, const Status& s) override {
+    if (status->ok()) *status = s;
+  }
+};
+
 // BlobFileSet is the set of all the blobs file generated by Titan.
 // It records blob file meta in terms of column family.
 class BlobFileSet {

src/blob_format.cc

@@ -300,6 +300,17 @@ TitanInternalStats::StatsType BlobFileMeta::GetDiscardableRatioLevel() const {
   return type;
 }
 
+void BlobFileMeta::Dump(bool with_keys) const {
+  fprintf(stdout, "file %" PRIu64 ", size %" PRIu64 ", level %" PRIu32,
+          file_number_, file_size_, file_level_);
+  if (with_keys) {
+    fprintf(stdout, ", smallest key: %s, largest key: %s",
+            Slice(smallest_key_).ToString(true /*hex*/).c_str(),
+            Slice(largest_key_).ToString(true /*hex*/).c_str());
+  }
+  fprintf(stdout, "\n");
+}
+
 void BlobFileHeader::EncodeTo(std::string* dst) const {
   PutFixed32(dst, kHeaderMagicNumber);
   PutFixed32(dst, version);

src/blob_format.h

@@ -288,6 +288,7 @@ class BlobFileMeta {
                 (file_size_ - kBlobMaxHeaderSize - kBlobFooterSize));
   }
   TitanInternalStats::StatsType GetDiscardableRatioLevel() const;
+  void Dump(bool with_keys) const;
 
  private:
   // Persistent field

src/edit_collector.h

 #pragma once
 
+#include <inttypes.h>
+
+#include <algorithm>
 #include <unordered_map>
 
 #include "blob_file_set.h"
 #include "util/string_util.h"
 #include "version_edit.h"
 
-#include <inttypes.h>
-
 namespace rocksdb {
 namespace titandb {
 
@@ -104,6 +105,17 @@ class EditCollector {
     return Status::Corruption("No next file number in manifest file");
   }
 
+  void Dump(bool with_keys) const {
+    if (has_next_file_number_) {
+      fprintf(stdout, "next_file_number: %" PRIu64 "\n", next_file_number_);
+      for (auto& cf : column_families_) {
+        fprintf(stdout, "column family: %" PRIu32 "\n", cf.first);
+        cf.second.Dump(with_keys);
+        fprintf(stdout, "\n");
+      }
+    }
+  }
+
  private:
   class CFEditCollector {
    public:
@@ -196,6 +208,31 @@ class EditCollector {
       return Status::OK();
     }
 
+    void Dump(bool with_keys) const {
+      std::vector<uint64_t> files;
+      files.reserve(added_files_.size());
+      for (auto& file : added_files_) {
+        files.push_back(file.first);
+      }
+      std::sort(files.begin(), files.end());
+      for (uint64_t file : files) {
+        if (deleted_files_.count(file) == 0) {
+          added_files_.at(file)->Dump(with_keys);
+        }
+      }
+      bool has_additional_deletion = false;
+      for (auto& file : deleted_files_) {
+        if (added_files_.count(file.first) == 0) {
+          if (!has_additional_deletion) {
+            fprintf(stdout, "additional deletion:\n");
+            has_additional_deletion = true;
+          }
+          fprintf(stdout, "file %" PRIu64 ", seq %" PRIu64 "\n", file.first,
+                  file.second);
+        }
+      }
+    }
+
    private:
     std::unordered_map<uint64_t, std::shared_ptr<BlobFileMeta>> added_files_;
     std::unordered_map<uint64_t, SequenceNumber> deleted_files_;
@@ -210,4 +247,4 @@ class EditCollector {
 };
 
 }  // namespace titandb
-}  // namespace rocksdb
\ No newline at end of file
+}  // namespace rocksdb

src/version_edit.cc

@@ -107,5 +107,25 @@ bool operator==(const VersionEdit& lhs, const VersionEdit& rhs) {
           lhs.deleted_files_ == rhs.deleted_files_);
 }
 
+void VersionEdit::Dump(bool with_keys) const {
+  fprintf(stdout, "column_family_id: %" PRIu32 "\n", column_family_id_);
+  if (has_next_file_number_) {
+    fprintf(stdout, "next_file_number: %" PRIu64 "\n", next_file_number_);
+  }
+  if (!added_files_.empty()) {
+    fprintf(stdout, "add files:\n");
+    for (auto& file : added_files_) {
+      file->Dump(with_keys);
+    }
+  }
+  if (!deleted_files_.empty()) {
+    fprintf(stdout, "delete files:\n");
+    for (auto& file : deleted_files_) {
+      fprintf(stdout, "file %" PRIu64 ", seq %" PRIu64 "\n", file.first,
+              file.second);
+    }
+  }
+}
+
 }  // namespace titandb
 }  // namespace rocksdb

src/version_edit.h

@@ -42,6 +42,8 @@ class VersionEdit {
 
   friend bool operator==(const VersionEdit& lhs, const VersionEdit& rhs);
 
+  void Dump(bool with_keys) const;
+
  private:
   friend class BlobFileSet;
   friend class VersionTest;

tools/manifest_dump.cc

+// Copyright 2021-present TiKV Project Authors. Licensed under Apache-2.0.
+
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+  fprintf(stderr, "Please install gflags to run Titan tools.\n");
+  return 1;
+}
+#else
+
+#include <memory>
+
+#include "edit_collector.h"
+#include "rocksdb/env.h"
+#include "util/file_reader_writer.h"
+#include "util/gflags_compat.h"
+#include "version_edit.h"
+
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+using GFLAGS_NAMESPACE::SetUsageMessage;
+
+DEFINE_string(path, "", "Path for Titan manifest file.");
+DEFINE_bool(ignore_tail_err, true,
+            "Ignore error encounter towards the tail of manifest.");
+DEFINE_bool(verbose, false, "Output each manifest record.");
+DEFINE_bool(with_keys, false, "Output blob file boundary keys");
+
+#define handle_error(s, location)                                           \
+  if (!s.ok()) {                                                            \
+    fprintf(stderr, "error when %s: %s\n", location, s.ToString().c_str()); \
+    return 1;                                                               \
+  }
+
+namespace rocksdb {
+namespace titandb {
+
+int manifest_dump() {
+  if (FLAGS_path.empty()) {
+    fprintf(stderr, "Manifest file path not given.\n");
+    return 1;
+  }
+  Env* env = Env::Default();
+  Status s;
+
+  // Open manifest file.
+  std::unique_ptr<SequentialFileReader> file_reader;
+  std::unique_ptr<SequentialFile> file;
+  s = env->NewSequentialFile(FLAGS_path, &file, EnvOptions());
+  handle_error(s, "open manifest file");
+  file_reader.reset(new SequentialFileReader(std::move(file), FLAGS_path));
+
+  // Open log reader.
+  LogReporter reporter;
+  reporter.status = &s;
+  log::Reader log_reader(nullptr, std::move(file_reader), &reporter,
+                         true /*checksum*/, 0 /*log_num*/);
+  Slice record;
+  std::string scratch;
+
+  // Loop through log records.
+  EditCollector edit_collector;
+  while (log_reader.ReadRecord(&record, &scratch) && s.ok()) {
+    VersionEdit edit;
+    s = DecodeInto(record, &edit);
+    handle_error(s, "parse version edit");
+    if (FLAGS_verbose) {
+      edit.Dump(FLAGS_with_keys);
+      fprintf(stdout, "\n");
+    }
+    s = edit_collector.AddEdit(edit);
+    handle_error(s, "colllect version edit");
+  }
+  if (!FLAGS_ignore_tail_err) {
+    handle_error(s, "parse manifest record");
+  }
+  edit_collector.Dump(FLAGS_with_keys);
+  return 0;
+}
+
+}  // namespace titandb
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  SetUsageMessage(std::string("\nUSAGE\n") + std::string(argv[0]) +
+                  " [OPTIONS]...");
+  ParseCommandLineFlags(&argc, &argv, true);
+  return rocksdb::titandb::manifest_dump();
+}
+
+#endif  // GFLAGS