Unverified Commit 68467cbb authored by yiwu-arbug's avatar yiwu-arbug Committed by GitHub

Cherry-pick build and dependency fixes (#325)

Cherry-picking the following changes that fixes zlib, jemalloc and gflags dependencies, and fixes a flaky test:
```
6ead62ca 2019-07-25 yiwu@pingcap.com     Avoid build require system zlib (#303)
9246b9c5 2019-07-25 hi@breeswish.org     Skip jemalloc options on specific platform (#324)
7a8dd731 2019-07-22 yiwu@pingcap.com     Add jemalloc-sys dependency when jemalloc is enabled (#320)
6f2c6320 2019-07-23 yiwu@pingcap.com     Not compile with gflags (#322)
09b80321 2019-07-22 yiwu@pingcap.com     Fix perf context test (#321)
d67f38e2 2019-07-21 yiwu@pingcap.com     Add jemalloc feature (#319)
```

Also update rocksdb to fix compile error with gcc9:
```
7a03c83ed 2019-07-23 psergey@askmonty.org Fix MyRocks compile warnings-treated-as-errors on Fedora 30, gcc 9.1.1 (#5553)
```
parent 13c1f945
...@@ -16,9 +16,10 @@ exclude = [ ...@@ -16,9 +16,10 @@ exclude = [
[features] [features]
default = [] default = []
valgrind = [] jemalloc = ["librocksdb_sys/jemalloc"]
portable = ["librocksdb_sys/portable"] portable = ["librocksdb_sys/portable"]
sse = ["librocksdb_sys/sse"] sse = ["librocksdb_sys/sse"]
valgrind = []
[dependencies] [dependencies]
libc = "0.2.11" libc = "0.2.11"
......
...@@ -14,6 +14,7 @@ tempdir = "0.3" ...@@ -14,6 +14,7 @@ tempdir = "0.3"
[features] [features]
default = [] default = []
jemalloc = ["jemalloc-sys"]
# portable doesn't require static link, though it's meaningless # portable doesn't require static link, though it's meaningless
# when not using with static-link right now in this crate. # when not using with static-link right now in this crate.
portable = ["libtitan_sys/portable"] portable = ["libtitan_sys/portable"]
...@@ -23,6 +24,10 @@ sse = ["libtitan_sys/sse"] ...@@ -23,6 +24,10 @@ sse = ["libtitan_sys/sse"]
cc = "1.0.3" cc = "1.0.3"
cmake = "0.1" cmake = "0.1"
[dependencies.jemalloc-sys]
version = ">= 0.1.8"
optional = true
[dependencies.libz-sys] [dependencies.libz-sys]
version = "1.0.25" version = "1.0.25"
features = ["static"] features = ["static"]
......
...@@ -19,6 +19,11 @@ use cmake::Config; ...@@ -19,6 +19,11 @@ use cmake::Config;
use std::path::PathBuf; use std::path::PathBuf;
use std::{env, str}; use std::{env, str};
// On these platforms jemalloc-sys will use a prefixed jemalloc which cannot be linked together
// with RocksDB.
// See https://github.com/gnzlbg/jemallocator/blob/bfc89192971e026e6423d9ee5aaa02bc56585c58/jemalloc-sys/build.rs#L45
const NO_JEMALLOC_TARGETS: &[&str] = &["android", "dragonfly", "musl", "darwin"];
fn main() { fn main() {
let mut build = build_rocksdb(); let mut build = build_rocksdb();
...@@ -71,19 +76,29 @@ fn link_cpp(build: &mut Build) { ...@@ -71,19 +76,29 @@ fn link_cpp(build: &mut Build) {
} }
fn build_rocksdb() -> Build { fn build_rocksdb() -> Build {
let mut build = Build::new(); let target = env::var("TARGET").expect("TARGET was not set");
for e in env::vars() {
println!("{:?}", e);
}
let mut cfg = Config::new("rocksdb"); let mut cfg = Config::new("rocksdb");
if cfg!(feature = "jemalloc") && NO_JEMALLOC_TARGETS.iter().all(|i| !target.contains(i)) {
cfg.register_dep("JEMALLOC").define("WITH_JEMALLOC", "ON");
}
if cfg!(feature = "portable") { if cfg!(feature = "portable") {
cfg.define("PORTABLE", "ON"); cfg.define("PORTABLE", "ON");
} }
if cfg!(feature = "sse") { if cfg!(feature = "sse") {
cfg.define("FORCE_SSE42", "ON"); cfg.define("FORCE_SSE42", "ON");
} }
// RocksDB cmake script expect libz.a being under ${DEP_Z_ROOT}/lib, but libz-sys crate put it
// under ${DEP_Z_ROOT}/build. Append the path to CMAKE_PREFIX_PATH to get around it.
env::set_var("CMAKE_PREFIX_PATH", {
let zlib_path = format!("{}/build", env::var("DEP_Z_ROOT").unwrap());
if let Ok(prefix_path) = env::var("CMAKE_PREFIX_PATH") {
format!("{};{}", prefix_path, zlib_path)
} else {
zlib_path
}
});
let dst = cfg let dst = cfg
.define("WITH_GFLAGS", "OFF")
.register_dep("Z") .register_dep("Z")
.define("WITH_ZLIB", "ON") .define("WITH_ZLIB", "ON")
.register_dep("BZIP2") .register_dep("BZIP2")
...@@ -95,8 +110,10 @@ fn build_rocksdb() -> Build { ...@@ -95,8 +110,10 @@ fn build_rocksdb() -> Build {
.register_dep("SNAPPY") .register_dep("SNAPPY")
.define("WITH_SNAPPY", "ON") .define("WITH_SNAPPY", "ON")
.build_target("rocksdb") .build_target("rocksdb")
.very_verbose(true)
.build(); .build();
let build_dir = format!("{}/build", dst.display()); let build_dir = format!("{}/build", dst.display());
let mut build = Build::new();
if cfg!(target_os = "windows") { if cfg!(target_os = "windows") {
let profile = match &*env::var("PROFILE").unwrap_or("debug".to_owned()) { let profile = match &*env::var("PROFILE").unwrap_or("debug".to_owned()) {
"bench" | "release" => "Release", "bench" | "release" => "Release",
......
extern crate cc; extern crate cc;
extern crate cmake; extern crate cmake;
use std::env;
fn main() { fn main() {
// RocksDB cmake script expect libz.a being under ${DEP_Z_ROOT}/lib, but libz-sys crate put it
// under ${DEP_Z_ROOT}/build. Append the path to CMAKE_PREFIX_PATH to get around it.
env::set_var("CMAKE_PREFIX_PATH", {
let zlib_path = format!("{}/build", env::var("DEP_Z_ROOT").unwrap());
if let Ok(prefix_path) = env::var("CMAKE_PREFIX_PATH") {
format!("{};{}", prefix_path, zlib_path)
} else {
zlib_path
}
});
let cur_dir = std::env::current_dir().unwrap(); let cur_dir = std::env::current_dir().unwrap();
let mut cfg = cmake::Config::new("titan"); let mut cfg = cmake::Config::new("titan");
if cfg!(feature = "portable") { if cfg!(feature = "portable") {
...@@ -25,6 +37,7 @@ fn main() { ...@@ -25,6 +37,7 @@ fn main() {
.register_dep("SNAPPY") .register_dep("SNAPPY")
.define("WITH_SNAPPY", "ON") .define("WITH_SNAPPY", "ON")
.build_target("titan") .build_target("titan")
.very_verbose(true)
.build(); .build();
println!("cargo:rustc-link-search=native={}/build", dst.display()); println!("cargo:rustc-link-search=native={}/build", dst.display());
println!("cargo:rustc-link-lib=static=titan"); println!("cargo:rustc-link-lib=static=titan");
......
Subproject commit f2140442071eb0e580a7c24b0a114eb5a77aa8f3 Subproject commit 7a03c83eddcafaa48c37a751045326deca9173ff
...@@ -419,6 +419,7 @@ mod test { ...@@ -419,6 +419,7 @@ mod test {
} }
} }
set_perf_level(PerfLevel::EnableCount);
let mut ctx = PerfContext::get(); let mut ctx = PerfContext::get();
let mut iter = db.iter(); let mut iter = db.iter();
......
...@@ -7,6 +7,7 @@ extern crate tempdir; ...@@ -7,6 +7,7 @@ extern crate tempdir;
mod test_column_family; mod test_column_family;
mod test_compact_range; mod test_compact_range;
mod test_compaction_filter; mod test_compaction_filter;
mod test_compression;
mod test_delete_files_in_range; mod test_delete_files_in_range;
mod test_delete_range; mod test_delete_range;
mod test_encryption; mod test_encryption;
......
// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
use rocksdb::{ColumnFamilyOptions, DBCompressionType, DBOptions, DB};
use tempdir::TempDir;
#[test]
// Make sure all compression types are supported.
fn test_compression() {
let path = TempDir::new("_rust_rocksdb_test_metadata").unwrap();
let compression_types = [
DBCompressionType::Snappy,
DBCompressionType::Zlib,
DBCompressionType::Bz2,
DBCompressionType::Lz4,
DBCompressionType::Lz4hc,
DBCompressionType::Zstd,
];
for compression_type in compression_types.iter() {
let mut opts = DBOptions::new();
opts.create_if_missing(true);
let mut cf_opts = ColumnFamilyOptions::new();
cf_opts.compression(*compression_type);
// DB open will fail if compression type is not supported.
DB::open_cf(
opts,
path.path().to_str().unwrap(),
vec![("default", cf_opts)],
)
.unwrap();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment