Add support for cloud data store (#517)

### Summary
Integration with rockset/rocksdb-cloud to add CloudEnv features for operating with cloud services like AWS S3 as a datastore.
- Created a sub-crate `librocksdb_cloud_sys` which contains the `rocksdb-cloud` sub-module containing code for managing cloud interactions, extracted from rockset/rocksdb-cloud and currently residing in [tikv/rocksdb~6.4.cloud](https://github.com/tikv/rocksdb/tree/6.4.cloud)
- Made necessary changes to [tikv/rocksdb~6.4.tikv](https://github.com/tikv/rocksdb/tree/6.4.tikv) that is housed in the sub-module `rocksdb` within the parent crate.
- Wrote CMakeLists.txt for use in the build process of `rocksdb_cloud` as a static library, usable for C-ABI based FFI calls. Added the above code to the build process of parent crate `librocksdb_sys` along with C-ABI binding code within `crocksdb`
- Created 'cloud.rs' and added `new_aws_env()` method to Env type of `rocksdb.rs` that merely makes a call to the `CloudEnv::NewAwsEnv()` via FFI for use as library features upstream. A test for the same was also added here.
- Used feature based conditional compile macros to all cloud interaction code.
- Made requisite changes to CI config for handling AWS SDK compilation.
- Added a sequence of steps to update submodule within Makefile

Solves #514

## Checkpoints

- [x] Added rocksdb-cloud as submodule in crate `librocksdb_cloud_sys`.
- [x] Write CMakeLists.txt to build only `cloud` code.
- [x] Generate C-bindings to interface with `NewAwsEnv` and `CloudEnvOptions`
- [x] Create `Env::new_aws_env` as an interface in rust
Signed-off-by: Devdutt Shenoi <devdutt@outlook.in>

.gitmodules

@@ -2,8 +2,13 @@
 	path = librocksdb_sys/rocksdb
 	url = https://github.com/tikv/rocksdb.git
 	branch = 6.4.tikv
+
 [submodule "titan"]
 	path = librocksdb_sys/libtitan_sys/titan
 	url = https://github.com/tikv/titan.git
 	branch = master
 
+[submodule "rocksdb-cloud"]
+	path = librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud
+	url = https://github.com/tikv/rocksdb
+	branch = 6.4.cloud

.travis.yml

 dist: bionic
 language: rust
 
-os:
-  - linux
-
 rust:
   - nightly
 
+cache:
+  cargo: false
+  directories:
+  - "/tmp/aws-sdk-cpp-1.8.14"
+
 addons:
   apt:
     sources:
@@ -19,6 +21,9 @@ jobs:
   - os: linux
     rust: nightly
     env: FEATURES="encryption,jemalloc,portable,sse"
+  - os: linux
+    rust: nightly
+    env: FEATURES="cloud"  
   - os: linux
     arch: arm64
     rust: nightly
@@ -31,14 +36,26 @@ env:
   global:
   - RUST_BACKTRACE=1
 
-cache: false
-
 before_script:
+  # Installing the AWS-SDK, so increase the maximum number of open file descriptors,
+  # since some tests use more FDs than the default limit.
+  - if [ "${FEATURES}" == "cloud" ]; then
+      echo "limit maxfiles 1024 unlimited" | sudo tee -a /etc/launchd.conf;
+      pushd /tmp;
+      wget https://github.com/aws/aws-sdk-cpp/archive/1.8.14.tar.gz -O /tmp/aws-sdk.tar.gz;
+      tar -xvf /tmp/aws-sdk.tar.gz > /dev/null;
+      popd;
+      pushd /tmp/aws-sdk-cpp-1.8.14;
+      cmake -DBUILD_ONLY='kinesis;core;s3;transfer' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TESTING=OFF .;
+      make -j4 all;
+      sudo make install;
+      popd;
+    fi
   - rustup component add rustfmt-preview
   - rustup component add clippy
 
 script:
-  # copile rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
+  # compiling rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
   - if [ "${TASK}" == "clang-format" ]; then
       bash ./travis-build/travis-format.sh;
     else

Cargo.toml

@@ -22,6 +22,8 @@ portable = ["librocksdb_sys/portable"]
 sse = ["librocksdb_sys/sse"]
 static_libcpp = ["librocksdb_sys/static_libcpp"]
 valgrind = []
+# rocksdb-cloud integration
+cloud = ["librocksdb_sys/cloud"]
 
 [dependencies]
 libc = "0.2.11"

Makefile

@@ -41,3 +41,14 @@ update_rocksdb:
 	fi
 	@git submodule sync
 	@git submodule update --init --remote librocksdb_sys/rocksdb
+
+update_rocksdb_cloud:
+	@if [ -n "${ROCKSDB_CLOUD_REPO}" ]; then \
+		git config --file=.gitmodules submodule.rocksdb-cloud.url https://github.com/${ROCKSDB_CLOUD_REPO}/rocksdb.git; \
+	fi
+	@if [ -n "${ROCKSDB_CLOUD_BRANCH}" ]; then \
+		git config --file=.gitmodules submodule.rocksdb-cloud.branch ${ROCKSDB_CLOUD_BRANCH}; \
+	fi
+	@git submodule sync
+	@git submodule update --init --remote librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud
+	
\ No newline at end of file

librocksdb_sys/Cargo.toml

@@ -8,6 +8,7 @@ links = "rocksdb"
 [dependencies]
 bzip2-sys = "0.1.8+1.0.8" 
 libc = "0.2.11"
+librocksdb_cloud_sys = { path = "librocksdb_cloud_sys" }
 libtitan_sys = { path = "libtitan_sys" }
 libz-sys = { version = "1.0.25", features = ["static"] }
 openssl-sys = { version = "0.9.54", optional = true, features = ["vendored"] }
@@ -25,6 +26,7 @@ jemalloc = ["tikv-jemalloc-sys"]
 portable = ["libtitan_sys/portable"]
 sse = ["libtitan_sys/sse"]
 static_libcpp = []
+cloud = []
 
 [build-dependencies]
 cc = "1.0.3"

librocksdb_sys/build.rs

@@ -128,6 +128,10 @@ fn link_cpp(build: &mut Build) {
 fn build_rocksdb() -> Build {
     let target = env::var("TARGET").expect("TARGET was not set");
     let mut cfg = Config::new("rocksdb");
+    // Conditionally compile with support for RocksDB-Cloud, setting USE_AWS
+    if cfg!(feature = "cloud") {
+        println!("cargo:rustc-link-lib=static=rocksdb_cloud");
+    }
     if cfg!(feature = "encryption") {
         cfg.register_dep("OPENSSL").define("WITH_OPENSSL", "ON");
         println!("cargo:rustc-link-lib=static=crypto");
@@ -195,6 +199,13 @@ fn build_rocksdb() -> Build {
     build.include(cur_dir.join("rocksdb"));
     build.include(cur_dir.join("libtitan_sys").join("titan").join("include"));
     build.include(cur_dir.join("libtitan_sys").join("titan"));
+    build.include(
+        cur_dir
+            .join("librocksdb_cloud_sys")
+            .join("rocksdb-cloud")
+            .join("include"),
+    );
+    build.include(cur_dir.join("librocksdb_cloud_sys").join("rocksdb-cloud"));
 
     // Adding rocksdb specific compile macros.
     // TODO: should make sure crocksdb compile options is the same as rocksdb and titan.
@@ -202,6 +213,9 @@ fn build_rocksdb() -> Build {
     if cfg!(feature = "encryption") {
         build.define("OPENSSL", None);
     }
+    if cfg!(feature = "cloud") {
+        build.define("USE_AWS", None).define("USE_CLOUD", None);
+    }
 
     println!("cargo:rustc-link-lib=static=rocksdb");
     println!("cargo:rustc-link-lib=static=titan");

librocksdb_sys/crocksdb/c.cc

@@ -15,6 +15,7 @@
 
 #include "db/column_family.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/cloud/cloud_env_options.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/convenience.h"
@@ -178,6 +179,9 @@ using rocksdb::titandb::TitanDBOptions;
 using rocksdb::titandb::TitanOptions;
 using rocksdb::titandb::TitanReadOptions;
 
+using rocksdb::CloudEnv;
+using rocksdb::CloudEnvOptions;
+
 using rocksdb::MemoryAllocator;
 
 #ifdef OPENSSL
@@ -5853,4 +5857,49 @@ void ctitandb_delete_files_in_ranges_cf(
   SaveError(errptr, static_cast<TitanDB*>(db->rep)->DeleteFilesInRanges(
                         cf->rep, &ranges[0], num_ranges, include_end));
 }
+
+/* RocksDB Cloud */
+#ifdef USE_CLOUD
+struct crocksdb_cloud_envoptions_t {
+  CloudEnvOptions rep;
+};
+
+crocksdb_env_t* crocksdb_cloud_aws_env_create(
+    crocksdb_env_t* base_env, const char* src_cloud_bucket,
+    const char* src_cloud_object, const char* src_cloud_region,
+    const char* dest_cloud_bucket, const char* dest_cloud_object,
+    const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options,
+    char** errptr) {
+  // Store a reference to a cloud env. A new cloud env object should be
+  // associated with every new cloud-db.
+  CloudEnv* cloud_env;
+
+  CloudEnv* cenv;
+  if (SaveError(errptr,
+                CloudEnv::NewAwsEnv(
+                    base_env->rep, src_cloud_bucket, src_cloud_object,
+                    src_cloud_region, dest_cloud_bucket, dest_cloud_object,
+                    dest_cloud_region, cloud_options->rep, nullptr, &cenv))) {
+    assert(cenv != nullptr);
+    return nullptr;
+  }
+  cloud_env = cenv;
+
+  crocksdb_env_t* result = new crocksdb_env_t;
+  result->rep = static_cast<Env*>(cloud_env);
+  result->block_cipher = nullptr;
+  result->encryption_provider = nullptr;
+  result->is_default = true;
+  return result;
+}
+
+crocksdb_cloud_envoptions_t* crocksdb_cloud_envoptions_create() {
+  crocksdb_cloud_envoptions_t* opt = new crocksdb_cloud_envoptions_t;
+  return opt;
+}
+
+void crocksdb_cloud_envoptions_destroy(crocksdb_cloud_envoptions_t* opt) {
+  delete opt;
+}
+#endif
 }  // end extern "C"

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -70,6 +70,7 @@ extern "C" {
 
 /* Exported types */
 
+typedef struct crocksdb_cloud_envoptions_t crocksdb_cloud_envoptions_t;
 typedef struct crocksdb_t crocksdb_t;
 typedef struct crocksdb_status_ptr_t crocksdb_status_ptr_t;
 typedef struct crocksdb_backup_engine_t crocksdb_backup_engine_t;
@@ -2348,6 +2349,21 @@ extern C_ROCKSDB_LIBRARY_API void ctitandb_delete_files_in_ranges_cf(
     const char* const* limit_keys, const size_t* limit_keys_lens,
     size_t num_ranges, unsigned char include_end, char** errptr);
 
+/* RocksDB Cloud */
+
+#ifdef USE_CLOUD
+extern C_ROCKSDB_LIBRARY_API crocksdb_env_t* crocksdb_cloud_aws_env_create(
+    crocksdb_env_t* base_env, const char* src_cloud_bucket,
+    const char* src_cloud_object, const char* src_cloud_region,
+    const char* dest_cloud_bucket, const char* dest_cloud_object,
+    const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options,
+    char** errptr);
+extern C_ROCKSDB_LIBRARY_API crocksdb_cloud_envoptions_t*
+crocksdb_cloud_envoptions_create();
+extern C_ROCKSDB_LIBRARY_API void crocksdb_cloud_envoptions_destroy(
+    crocksdb_cloud_envoptions_t* opt);
+#endif
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif

librocksdb_sys/librocksdb_cloud_sys/CMakeLists.txt

+# Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+cmake_minimum_required(VERSION 3.0)
+project(rocksdb_cloud)
+enable_language(CXX)
+enable_language(C)
+
+# Manage absence of rocksdb code dependence
+
+if (NOT ROCKSDB_GIT_REPO)
+  set(ROCKSDB_GIT_REPO "https://github.com/tikv/rocksdb.git")
+endif()
+
+if (NOT ROCKSDB_GIT_BRANCH)
+  set(ROCKSDB_GIT_BRANCH "6.4.tikv")
+endif()
+
+if (NOT DEFINED ROCKSDB_DIR)
+  if (GIT_FOUND)
+    if (WIN32)
+      execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO})
+    else()
+      execute_process(COMMAND ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO})
+    endif()
+    set(ROCKSDB_DIR "${CMAKE_BINARY_DIR}/rocksdb")
+  endif()
+endif()
+
+if (NOT DEFINED ROCKSDB_DIR)
+  message(FATAL_ERROR "ROCKSDB_DIR is not defined.")
+endif()
+
+# Append rocksdb modules
+list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DIR}/cmake/modules/")
+include("cmake_modules/rocksdb_flags.cmake")
+
+include_directories("${ROCKSDB_DIR}")
+include_directories("${ROCKSDB_DIR}/include")
+include_directories("rocksdb-cloud")
+include_directories("rocksdb-cloud/include")
+
+file(GLOB SOURCES rocksdb-cloud/cloud/*.cc)
+
+add_library(rocksdb_cloud STATIC ${SOURCES} $<TARGET_OBJECTS:cloud_env>)
+
+option(WITH_CLOUD_TESTS "Build with tests." ON)
+
+include_directories(SYSTEM ${ROCKSDB_DIR}/third-party/gtest-1.7.0/fused-src)
+
+add_library(cloud_env OBJECT "rocksdb-cloud/cloud/cloud_env.cc")
+
+include(GNUInstallDirs)
+install(DIRECTORY include/rocksdb/cloud
+  COMPONENT devel
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)
+install(TARGETS rocksdb_cloud
+  COMPONENT devel
+  ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+  INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)

librocksdb_sys/librocksdb_cloud_sys/Cargo.toml

+[package]
+name = "librocksdb_cloud_sys"
+version = "0.1.0"
+edition = "2018"
+build = "build.rs"
+links = "rocksdb-cloud"
+
+[features]
+cloud = []
+
+[build-dependencies]
+cc = "1.0.3"
+cmake = "0.1"

librocksdb_sys/librocksdb_cloud_sys/build.rs

+// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+extern crate cc;
+extern crate cmake;
+
+use std::env;
+
+fn main() {
+    let cur_dir = env::current_dir().unwrap();
+    let mut cfg = cmake::Config::new(".");
+    let dst = cfg
+        .define("ROCKSDB_DIR", cur_dir.join("..").join("rocksdb"))
+        .env("USE_AWS", "1")
+        .build_target("rocksdb_cloud")
+        .very_verbose(true)
+        .build();
+    println!("cargo:rustc-link-search=native={}/build", dst.display());
+    println!("cargo:rustc-link-lib=static=rocksdb_cloud");
+}

rocksdb-cloud @ ac0e18f4

+Subproject commit ac0e18f441de3e265c9f15d69dec22220f916814

librocksdb_sys/librocksdb_cloud_sys/src/lib.rs

+// The library requires this file for `cargo build`

rocksdb @ f114e136

-Subproject commit fa9b061b8717cbecbae370762c097b41f32107aa
+Subproject commit f114e1361188c41f7612ba7ee575d02b88dbd80c

librocksdb_sys/src/lib.rs

@@ -46,6 +46,8 @@ use libc::{c_char, c_double, c_int, c_uchar, c_void, size_t};
 #[repr(C)]
 pub struct Options(c_void);
 #[repr(C)]
+pub struct CloudEnvOptions(c_void);
+#[repr(C)]
 pub struct ColumnFamilyDescriptor(c_void);
 #[repr(C)]
 pub struct DBInstance(c_void);
@@ -2318,6 +2320,26 @@ extern "C" {
     );
 }
 
+// RocksDB Cloud
+extern "C" {
+    // NewAWSEnv
+    pub fn crocksdb_cloud_aws_env_create(
+        base_env: *mut DBEnv,
+        src_cloud_bucket: *const c_char,
+        src_cloud_object: *const c_char,
+        src_cloud_region: *const c_char,
+        dest_cloud_bucket: *const c_char,
+        dest_cloud_object: *const c_char,
+        dest_cloud_region: *const c_char,
+        opts: *mut CloudEnvOptions,
+        err: *mut *mut c_char,
+    ) -> *mut DBEnv;
+
+    // CloudEnvOptions
+    pub fn crocksdb_cloud_envoptions_create() -> *mut CloudEnvOptions;
+    pub fn crocksdb_cloud_envoptions_destroy(opt: *mut CloudEnvOptions);
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

src/cloud.rs

+// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+
+use crocksdb_ffi::{self};
+
+/// Options while opening a file in the cloud to read/write
+pub struct CloudEnvOptions {
+    pub inner: *mut crocksdb_ffi::CloudEnvOptions,
+}
+
+impl CloudEnvOptions {
+    pub fn new() -> CloudEnvOptions {
+        unsafe {
+            CloudEnvOptions {
+                inner: crocksdb_ffi::crocksdb_cloud_envoptions_create(),
+            }
+        }
+    }
+}
+
+impl Drop for CloudEnvOptions {
+    fn drop(&mut self) {
+        unsafe {
+            crocksdb_ffi::crocksdb_cloud_envoptions_destroy(self.inner);
+        }
+    }
+}

src/lib.rs

@@ -24,6 +24,8 @@ pub extern crate librocksdb_sys;
 #[cfg(test)]
 extern crate tempfile;
 
+#[cfg(feature = "cloud")]
+pub use cloud::CloudEnvOptions;
 pub use compaction_filter::{
     new_compaction_filter, new_compaction_filter_factory, new_compaction_filter_raw,
     CompactionFilter, CompactionFilterContext, CompactionFilterFactory,
@@ -69,6 +71,8 @@ pub use titan::{TitanBlobIndex, TitanDBOptions};
 #[allow(deprecated)]
 pub use rocksdb::Kv;
 
+#[cfg(feature = "cloud")]
+mod cloud;
 mod compaction_filter;
 pub mod comparator;
 #[cfg(feature = "encryption")]

src/rocksdb.rs

@@ -39,6 +39,8 @@ use std::str::from_utf8;
 use std::sync::Arc;
 use std::{fs, ptr, slice};
 
+#[cfg(feature = "cloud")]
+use cloud::CloudEnvOptions;
 #[cfg(feature = "encryption")]
 use encryption::{DBEncryptionKeyManager, EncryptionKeyManager};
 use table_properties::{TableProperties, TablePropertiesCollection};
@@ -2543,6 +2545,47 @@ impl Env {
         }
     }
 
+    // Create an cloud env to operate with AWS S3.
+    #[cfg(feature = "cloud")]
+    pub fn new_aws_env(
+        base_env: Arc<Env>,
+        src_cloud_bucket: &str,
+        src_cloud_object: &str,
+        src_cloud_region: &str,
+        dest_cloud_bucket: &str,
+        dest_cloud_object: &str,
+        dest_cloud_region: &str,
+        opts: CloudEnvOptions,
+    ) -> Result<Env, String> {
+        let mut err = ptr::null_mut();
+        let src_cloud_bucket = CString::new(src_cloud_bucket).unwrap();
+        let src_cloud_object = CString::new(src_cloud_object).unwrap();
+        let src_cloud_region = CString::new(src_cloud_region).unwrap();
+        let dest_cloud_bucket = CString::new(dest_cloud_bucket).unwrap();
+        let dest_cloud_object = CString::new(dest_cloud_object).unwrap();
+        let dest_cloud_region = CString::new(dest_cloud_region).unwrap();
+        let env = unsafe {
+            crocksdb_ffi::crocksdb_cloud_aws_env_create(
+                base_env.inner,
+                src_cloud_bucket.as_ptr(),
+                src_cloud_object.as_ptr(),
+                src_cloud_region.as_ptr(),
+                dest_cloud_bucket.as_ptr(),
+                dest_cloud_object.as_ptr(),
+                dest_cloud_region.as_ptr(),
+                opts.inner,
+                &mut err,
+            )
+        };
+        if !err.is_null() {
+            return Err(unsafe { crocksdb_ffi::error_message(err) });
+        }
+        Ok(Env {
+            inner: env,
+            base: Some(base_env),
+        })
+    }
+
     // Create a ctr encrypted env with a given base env and a given ciper text.
     // The length of ciper text must be 2^n, and must be less or equal to 2048.
     // The recommanded block size are 1024, 512 and 256.
@@ -3444,4 +3487,22 @@ mod test {
         let first_path = db.get_db_options().get_db_path(0).unwrap();
         assert_eq!(path, first_path.as_str());
     }
+
+    #[cfg(feature = "cloud")]
+    #[test]
+    fn test_cloud_aws_env_creation() {
+        let k_db_path = "/tmp/rocksdb_main_db";
+        let k_bucket_suffix = "cloud.clone.example.";
+        let k_region = "us-west-2";
+        let _db = Env::new_aws_env(
+            Arc::new(Env::default()),
+            &k_bucket_suffix,
+            &k_db_path,
+            &k_region,
+            &k_bucket_suffix,
+            &k_db_path,
+            &k_region,
+            CloudEnvOptions::new(),
+        );
+    }
 }

tests/cases/test_rocksdb_options.rs

@@ -175,7 +175,7 @@ fn test_set_ratelimiter_with_auto_tuned() {
         100 * 1024 * 1024,
         10 * 100000,
         DBRateLimiterMode::AllIo,
-        true
+        true,
     );
     let db = DB::open(opts, path.path().to_str().unwrap()).unwrap();
     drop(db);