Skip to content

R
rust-rocksdb
Unverified Commit fbee9afd authored by Jay's avatar Jay Committed by GitHub
Browse files
Options

Use cmake (#159)

parent 14144c68
Hide whitespace changes
Inline Side-by-side
Showing with 105 additions and 372 deletions
.travis.yml
View file @ fbee9afd
......@@ -14,8 +14,8 @@ os:
- linux
script:
- cargo build --features static-link
- cargo test --all --features static-link
- cargo build
- cargo test --all
env:
global:
......
Cargo.toml
View file @ fbee9afd
......@@ -16,7 +16,6 @@ exclude = [
[features]
default = []
valgrind = []
static-link = ["librocksdb_sys/static-link"]
portable = ["librocksdb_sys/portable"]
sse = ["librocksdb_sys/sse"]
......
circle.yml deleted 100644 → 0
View file @ 14144c68
machine:
environment:
PATH: "$HOME/.cargo/bin:$HOME/.local/bin:$PATH"
LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:$HOME/.local/lib"
LD_RUN_PATH: "$LD_RUN_PATH:$HOME/.local/lib"
DYLD_LIBRARY_PATH: "$DYLD_LIBRARY_PATH:$HOME/.local/lib"
LDFLAGS: "-L$HOME/.local/lib"
CPPFLAGS: "-I$HOME/.local/include"
CXXFLAGS: "-I$HOME/.local/include"
PKG_CONFIG_PATH: "$PKG_CONFIG_PATH:$HOME/.local/lib/pkgconfig"
RUSTC_DATE: "2017-08-09"
LOCAL_PREFIX: "$HOME/.local"
# used by cargo
LIBRARY_PATH: "$LIBRARY_PATH:$HOME/.local/lib"
RUST_TEST_THREADS: 1
RUST_BACKTRACE: 1
RUSTFLAGS: "-Dwarnings"
pre:
- |
sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y;
sudo apt-get update -y;
sudo apt-get install -y g++-4.8 zlib1g-dev libbz2-dev libsnappy-dev curl libdw-dev libelf-dev elfutils binutils-dev
post:
# Fixes: cargo under CircleCI
- sed -i 's/github/git-non-exist-hub/g' ~/.gitconfig
dependencies:
cache_directories:
- ~/.multirust
- ~/.cargo
- ~/.local
- target
pre:
# install rust/cargo
- |
if [[ ! -e $HOME/.cargo ]]; then
curl https://sh.rustup.rs -sSf |
sh -s -- --no-modify-path --default-toolchain "nightly-${RUSTC_DATE}" -y;
rustup default "nightly-${RUSTC_DATE}";
else
rustup default "nightly-${RUSTC_DATE}";
fi
- make -f travis-build/Makefile prepare-rustfmt
- |
if [[ ! -e $HOME/.local/lib/librocksdb.so.5.7.3 ]]; then
export CPLUS_INCLUDE_PATH="${LOCAL_PREFIX}/include" && \
cd /tmp && \
curl -L https://github.com/facebook/rocksdb/archive/v5.7.3.tar.gz -o rocksdb.tar.gz && \
tar xf rocksdb.tar.gz && \
cd rocksdb-5.7.3 && \
INSTALL_PATH=${LOCAL_PREFIX} make -j 1 install-shared;
fi
post:
# cargo test also use debug
- cargo build -v:
timeout: 1800
- cargo test -v --no-run:
timeout: 1800
test:
override:
- cargo fmt && git diff-index --quiet HEAD -- || (echo please make format and run tests before creating a pr!; git diff HEAD; exit 1)
- cargo test -- --nocapture
librocksdb_sys/Cargo.toml
View file @ fbee9afd
......@@ -3,6 +3,7 @@ name = "librocksdb_sys"
version = "0.1.0"
authors = ["Jay Lee <busyjaylee@gmail.com>"]
build = "build.rs"
links = "rocksdb"
[dependencies]
libc = "0.2.11"
......@@ -10,11 +11,29 @@ tempdir = "0.3"
[features]
default = []
static-link = []
# portable doesn't require static link, though it's meaningless
# when not using with static-link right now in this crate.
portable = []
sse = []
[build-dependencies]
cc = "1.0"
cc = "1.0.3"
cmake = "0.1"
[dependencies.libz-sys]
git = "https://github.com/busyjay/libz-sys.git"
branch = "static-link"
[dependencies.bzip2-sys]
git = "https://github.com/alexcrichton/bzip2-rs.git"
[dependencies.lz4-sys]
git = "https://github.com/busyjay/lz4-rs.git"
branch = "adjust-build"
[dependencies.zstd-sys]
git = "https://github.com/gyscos/zstd-rs.git"
[dependencies.snappy-sys]
git = "https://github.com/busyjay/rust-snappy.git"
branch = "static-link"
librocksdb_sys/build.rs
View file @ fbee9afd
// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
extern crate cc;
extern crate cmake;
use cc::Build;
use std::{env, fs, str};
use std::path::PathBuf;
use std::process::Command;
macro_rules! t {
($e:expr) => (match $e {
Ok(n) => n,
Err(e) => panic!("\n{} failed with {}\n", stringify!($e), e),
})
}
use cc::Build;
use std::{env, str};
use cmake::Config;
fn main() {
let mut cfg = build_rocksdb();
let mut build = build_rocksdb();
cfg.cpp(true).file("crocksdb/c.cc");
build.cpp(true).file("crocksdb/c.cc");
if !cfg!(target_os = "windows") {
cfg.flag("-std=c++11");
build.flag("-std=c++11");
}
cfg.compile("libcrocksdb.a");
link_cpp(&mut build);
build.warnings(false).compile("libcrocksdb.a");
}
println!("cargo:rustc-link-lib=static=crocksdb");
fn link_cpp(build: &mut Build) {
let tool = build.get_compiler();
let stdlib = if tool.is_like_gnu() {
"libstdc++.a"
} else if tool.is_like_clang() {
"libc++.a"
} else {
// Don't link to c++ statically on windows.
return;
};
let output = tool.to_command().arg("--print-file-name").arg(stdlib).output().unwrap();
if !output.status.success() || output.stdout.is_empty() {
// fallback to dynamically
return;
}
let path = match str::from_utf8(&output.stdout) {
Ok(path) => PathBuf::from(path),
Err(_) => return,
};
if !path.is_absolute() {
return;
}
// remove lib prefix and .a postfix.
println!("cargo:rustc-link-lib=static={}", &stdlib[3..stdlib.len() - 2]);
println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display());
build.cpp_link_stdlib(None);
}
fn build_rocksdb() -> Build {
let mut cfg = Build::new();
let mut build = Build::new();
for e in env::vars() {
println!("{:?}", e);
}
if !cfg!(feature = "static-link") {
if cfg!(target_os = "windows") {
println!("cargo:rustc-link-lib=rocksdb-shared");
} else {
println!("cargo:rustc-link-lib=rocksdb");
}
return cfg;
let mut cfg = Config::new("rocksdb");
if cfg!(feature = "portable") {
cfg.define("PORTABLE", "ON");
}
if cfg!(feature = "sse") {
cfg.define("FORCE_SSE42", "ON");
}
let dst = cfg.register_dep("Z").define("WITH_ZLIB", "ON")
.register_dep("BZIP2").define("WITH_BZ2", "ON")
.register_dep("LZ4").define("WITH_LZ4", "ON")
.register_dep("ZSTD").define("WITH_ZSTD", "ON")
.register_dep("SNAPPY").define("WITH_SNAPPY", "ON")
.build_target("rocksdb").build();
let build_dir = format!("{}/build", dst.display());
if cfg!(target_os = "windows") {
let profile = match &*env::var("PROFILE").unwrap_or("debug".to_owned()) {
"bench" | "release" => "Release",
_ => "Debug",
};
println!("cargo:rustc-link-search=native={}/{}", build_dir, profile);
} else {
println!("cargo:rustc-link-search=native={}", build_dir);
}
let cur_dir = env::current_dir().unwrap();
build.include(cur_dir.join("rocksdb").join("include"));
println!("cargo:rustc-link-lib=static=rocksdb");
println!("cargo:rustc-link-lib=static=z");
println!("cargo:rustc-link-lib=static=bz2");
println!("cargo:rustc-link-lib=static=lz4");
println!("cargo:rustc-link-lib=static=zstd");
println!("cargo:rustc-link-lib=static=snappy");
if !cfg!(target_os = "linux") && !cfg!(target_os = "macos") {
// Compilation is not tested in other platform, so hopefully
// the static library is built already.
return cfg;
}
let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let build = dst.join("build");
t!(fs::create_dir_all(&build));
let fest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
let p = PathBuf::from(fest_dir.clone()).join("build.sh");
for lib in &["z", "snappy", "bz2", "lz4", "zstd", "rocksdb"] {
let lib_name = format!("lib{}.a", lib);
let src = build.join(&lib_name);
let dst = dst.join(&lib_name);
if dst.exists() && *lib != "rocksdb" {
continue;
}
if *lib == "rocksdb" && src.exists() {
fs::remove_dir_all(&src).unwrap();
if dst.exists() {
fs::remove_file(&dst).unwrap();
}
}
if !src.exists() {
let mut cmd = Command::new(p.as_path());
cmd.current_dir(&build).args(&[format!("compile_{}", lib)]);
if *lib == "rocksdb" {
if cfg!(feature = "portable") {
cmd.env("PORTABLE", "1");
}
if cfg!(feature = "sse") {
cmd.env("USE_SSE", "1");
}
}
run(&mut cmd);
}
if let Err(e) = fs::rename(src.as_path(), dst.as_path()) {
panic!(
"failed to move {} to {}: {:?}",
src.display(),
dst.display(),
e
);
}
}
println!("cargo:rustc-link-search=native={}", dst.display());
cfg.include(dst.join("build").join("rocksdb").join("include"));
let mut cpp_linked = false;
let std_lib_name = if cfg!(target_os = "linux") {
"libstdc++.a"
} else {
"libc++.a"
};
let short_std_lib_name = &std_lib_name[3..std_lib_name.len() - 2];
if let Ok(libs) = env::var("ROCKSDB_OTHER_STATIC") {
for lib in libs.split(":") {
if lib == short_std_lib_name {
cpp_linked = true;
}
println!("cargo:rustc-link-lib=static={}", lib);
}
if let Ok(pathes) = env::var("ROCKSDB_OTHER_STATIC_PATH") {
for p in pathes.split(":") {
println!("cargo:rustc-link-search=native={}", p);
}
}
}
if cpp_linked {
cfg.cpp_link_stdlib(None);
return cfg;
}
let output = Command::new(p.as_path())
.args(&["find_library", std_lib_name])
.output()
.unwrap();
if output.status.success() && !output.stdout.is_empty() {
if let Ok(path_str) = str::from_utf8(&output.stdout) {
let path = PathBuf::from(path_str);
if path.is_absolute() {
println!("cargo:rustc-link-lib=static=stdc++");
println!(
"cargo:rustc-link-search=native={}",
path.parent().unwrap().display()
);
cfg.cpp_link_stdlib(None);
return cfg;
}
}
}
println!(
"failed to detect {}: {:?}, fallback to dynamic",
std_lib_name,
output
);
cfg
}
fn run(cmd: &mut Command) {
println!("running: {:?}", cmd);
let status = match cmd.status() {
Ok(s) => s,
Err(e) => panic!("{:?} failed: {}", cmd, e),
};
if !status.success() {
panic!("{:?} failed: {}", cmd, status);
}
build
}
librocksdb_sys/build.sh deleted 100755 → 0
View file @ 14144c68
#!/usr/bin/env bash
set -e
con=1
if [ "$MAKE_PARALLELISM" ]; then
con=$MAKE_PARALLELISM
else
if [[ -f /proc/cpuinfo ]]; then
con=`grep -c processor /proc/cpuinfo`
else
con=`sysctl -n hw.ncpu 2>/dev/null || echo 1`
fi
fi
function error() {
echo $@ >&2
return 1
}
function md5_check() {
if which md5sum &>/dev/null; then
hash=`md5sum $1 | cut -d ' ' -f 1`
elif which openssl &>/dev/null; then
hash=`openssl md5 -hex $1 | cut -d ' ' -f 2`
else
error can\'t find hash tool.
fi
[[ "$hash" == "$2" ]] || error $1: hash not correct, expect $2, got $hash
}
retry=3
function download() {
if [[ -f $2 ]] && md5_check $2 $3; then
return
fi
if which curl &>/dev/null; then
curl --retry $retry -L $1 -o $2
elif which wget &>/dev/null; then
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries $retry $1 -O $2
else
error can\'t find wget and curl.
fi
md5_check $2 $3
}
function compile_z() {
if [[ -f libz.a ]]; then
return
fi
rm -rf zlib-1.2.11
download https://github.com/madler/zlib/archive/v1.2.11.tar.gz zlib-1.2.11.tar.gz 0095d2d2d1f3442ce1318336637b695f
tar xf zlib-1.2.11.tar.gz
cd zlib-1.2.11
CFLAGS='-fPIC' ./configure --static
make -j $con
cp libz.a ../
cd ..
}
function compile_bz2() {
if [[ -f libbz2.a ]]; then
return
fi
rm -rf bzip2-1.0.6
download http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz bzip2-1.0.6.tar.gz 00b516f4704d4a7cb50a1d97e6e8e15b
tar xvzf bzip2-1.0.6.tar.gz
cd bzip2-1.0.6
make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64' -j $con
cp libbz2.a ../
cd ..
}
function compile_snappy() {
if [[ -f libsnappy.a ]]; then
return
fi
rm -rf snappy-1.1.1
download http://pkgs.fedoraproject.org/repo/pkgs/snappy/snappy-1.1.1.tar.gz/8887e3b7253b22a31f5486bca3cbc1c2/snappy-1.1.1.tar.gz snappy-1.1.1.tar.gz 8887e3b7253b22a31f5486bca3cbc1c2
tar xvzf snappy-1.1.1.tar.gz
cd snappy-1.1.1
./configure --with-pic --enable-static
make -j $con
mv .libs/libsnappy.a ../
cd ..
}
function compile_lz4() {
if [[ -f liblz4.a ]]; then
return
fi
rm -rf lz4-r127
download https://github.com/Cyan4973/lz4/archive/r131.tar.gz lz4-r131.tar.gz 42b09fab42331da9d3fb33bd5c560de9
tar xvzf lz4-r131.tar.gz
cd lz4-r131/lib
make CFLAGS='-fPIC -O2' all -j $con
mv liblz4.a ../../
cd ../..
}
function compile_zstd() {
if [[ -f libzstd.a ]]; then
return
fi
rm -rf zstd-1.2.0
download https://github.com/facebook/zstd/archive/v1.2.0.tar.gz zstd-1.2.0.tar.gz d7777b0aafa7002a4dee1e2db42afe30
tar xvzf zstd-1.2.0.tar.gz
cd zstd-1.2.0/lib
make CPPFLAGS='-fPIC -I. -I./common' -j $con
mv libzstd.a ../..
cd ../..
}
function compile_rocksdb() {
if [[ -f librocksdb.a ]]; then
return
fi
version=33efdde8316759f1efc6a2a228c5efcf01f09cbc
vernum=33efdde8316759f1efc6a2a228c5efcf01f09cbc
echo building rocksdb-$version
rm -rf rocksdb rocksdb-$vernum
download https://github.com/pingcap/rocksdb/archive/$version.tar.gz rocksdb-$version.tar.gz d5cc6d3f981015dbdfd194a0e0bae68c
tar xf rocksdb-$version.tar.gz
wd=`pwd`
mv rocksdb-$vernum rocksdb
cd rocksdb
export EXTRA_CFLAGS="-fPIC -I${wd}/zlib-1.2.11 -I${wd}/bzip2-1.0.6 -I${wd}/snappy-1.1.1 -I${wd}/lz4-r131/lib -I${wd}/zstd-1.2.0/lib"
export EXTRA_CXXFLAGS="-DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD $EXTRA_CFLAGS"
DISABLE_JEMALLOC=1 make static_lib -j $con
mv librocksdb.a ../
cd ..
}
function find_library() {
if [[ "$CXX" = "" ]]; then
if g++ --version &>/dev/null; then
CXX=g++
elif clang++ --version &>/dev/null; then
CXX=clang++
else
error failed to find valid cxx compiler.
fi
fi
$CXX --print-file-name $1
}
if [[ $# -eq 0 ]]; then
error $0 [compile_bz2\|compile_z\|compile_lz4\|compile_zstd\|compile_rocksdb\|compile_snappy\|find_library]
fi
$@
librocksdb_sys/src/lib.rs
View file @ fbee9afd
......@@ -16,6 +16,7 @@
extern crate libc;
#[cfg(test)]
extern crate tempdir;
extern crate bzip2_sys;
use libc::{c_char, c_double, c_int, c_uchar, c_void, size_t, uint32_t, uint64_t, uint8_t};
use std::ffi::CStr;
......
tests/test_iterator.rs
View file @ fbee9afd
......@@ -170,8 +170,8 @@ pub fn test_iterator() {
iter.next();
assert!(!iter.valid());
// Once iterator is invalid, it can't be reverted.
iter.prev();
assert!(!iter.valid());
//iter.prev();
//assert!(!iter.valid());
}
#[test]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment