diff --git a/.travis.yml b/.travis.yml index 167dc2622..8d2349dae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,12 +44,12 @@ after_success: | wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && make install DESTDIR=../tmp && cd ../.. && cargo test --no-run ${KCOV_FEATURES} ${TARGETS} && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_util-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethash-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethsync-* && - ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_rpc-* && - ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /.cargo,/root/.multirust target/kcov target/debug/parity-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_util-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethash-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethsync-* && + ./kcov-master/tmp/usr/local/bin/kcov --exclude-pattern /usr/,/.cargo,/root/.multirust,src/tests,util/json-tests,util/src/network/tests,sync/src/tests,ethcore/src/tests,ethcore/src/evm/tests target/kcov target/debug/deps/ethcore_rpc-* && + ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /usr/,/.cargo,/root/.multirust target/kcov target/debug/parity-* && [ $TRAVIS_BRANCH = master ] && [ $TRAVIS_PULL_REQUEST = false ] && [ $TRAVIS_RUST_VERSION = beta ] && diff --git a/Cargo.lock b/Cargo.lock index bcbec5408..329d8217a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,7 +74,7 @@ name = "clippy" version = "0.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex-syntax 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -130,7 +130,7 @@ name = "docopt" version = "0.6.78" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -146,7 +146,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -155,7 +155,7 @@ version = "0.5.4" source = "git+https://github.com/arkpar/rust-secp256k1.git#45503e1de68d909b1862e3f2bdb9e1cdfdff3f1e" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -186,7 +186,6 @@ dependencies = [ "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -229,14 +228,14 @@ dependencies = [ "ethcore-devtools 0.9.99", "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "json-tests 0.1.0", "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rocksdb 0.4.1 (git+https://github.com/arkpar/rust-rocksdb.git)", "rust-crypto 0.2.34 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -272,7 +271,7 @@ dependencies = [ [[package]] name = "gcc" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -291,7 +290,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -323,7 +322,7 @@ dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -343,7 +342,7 @@ dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -354,14 +353,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.53 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "xml-rs 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)", "xmltree 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "itertools" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -390,7 +389,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -427,6 +426,15 @@ name = "libc" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "librocksdb-sys" +version = "0.2.1" +source = "git+https://github.com/arkpar/rust-rocksdb.git#2156621f583bda95c1c07e89e79e4019f75158ee" +dependencies = [ + "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "log" version = "0.3.5" @@ -466,7 +474,7 @@ dependencies = [ "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -479,14 +487,14 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "net2" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -515,7 +523,7 @@ dependencies = [ [[package]] name = "nom" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -548,6 +556,11 @@ name = "odds" version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pkg-config" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "primal" version = "0.2.3" @@ -617,26 +630,27 @@ dependencies = [ [[package]] name = "regex" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rocksdb" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.4.1" +source = "git+https://github.com/arkpar/rust-rocksdb.git#2156621f583bda95c1c07e89e79e4019f75158ee" dependencies = [ - "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "librocksdb-sys 0.2.1 (git+https://github.com/arkpar/rust-rocksdb.git)", ] [[package]] @@ -644,7 +658,7 @@ name = "rust-crypto" version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -674,7 +688,7 @@ name = "semver" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -710,7 +724,7 @@ dependencies = [ name = "sha3" version = "0.1.0" dependencies = [ - "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -794,7 +808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unicase" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 660fdb1b2..c3a3d32dc 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -10,7 +10,6 @@ authors = ["Ethcore "] log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" -rocksdb = "0.3" heapsize = "0.3" rust-crypto = "0.2.34" time = "0.1" diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index b95b659ee..30f039600 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -17,7 +17,6 @@ //! Blockchain database. use util::*; -use rocksdb::{DB, WriteBatch, Writable}; use header::*; use extras::*; use transaction::*; @@ -118,7 +117,7 @@ struct ExtrasUpdate { /// Block hash. hash: H256, /// DB update batch. - batch: WriteBatch, + batch: DBTransaction, /// Inserted block familial details. details: BlockDetails, /// New best block (if it has changed). @@ -248,8 +247,8 @@ pub struct BlockChain { blocks_blooms: RwLock>, block_receipts: RwLock>, - extras_db: DB, - blocks_db: DB, + extras_db: Database, + blocks_db: Database, cache_man: RwLock, @@ -331,12 +330,12 @@ impl BlockChain { // open extras db let mut extras_path = path.to_path_buf(); extras_path.push("extras"); - let extras_db = DB::open_default(extras_path.to_str().unwrap()).unwrap(); + let extras_db = Database::open_default(extras_path.to_str().unwrap()).unwrap(); // open blocks db let mut blocks_path = path.to_path_buf(); blocks_path.push("blocks"); - let blocks_db = DB::open_default(blocks_path.to_str().unwrap()).unwrap(); + let blocks_db = Database::open_default(blocks_path.to_str().unwrap()).unwrap(); let mut cache_man = CacheManager{cache_usage: VecDeque::new(), in_use: HashSet::new()}; (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); @@ -377,7 +376,7 @@ impl BlockChain { bc.blocks_db.put(&hash, genesis).unwrap(); - let batch = WriteBatch::new(); + let batch = DBTransaction::new(); batch.put_extras(&hash, &details); batch.put_extras(&header.number(), &hash); batch.put(b"best", &hash).unwrap(); @@ -571,7 +570,7 @@ impl BlockChain { }; // prepare the batch - let batch = WriteBatch::new(); + let batch = DBTransaction::new(); // insert new block details batch.put_extras(&hash, &details); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 66f6739a9..cb919c05a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -18,7 +18,6 @@ use util::*; use util::panics::*; -use rocksdb::{Options, DB, DBCompactionStyle}; use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; @@ -197,7 +196,7 @@ pub struct Client { } const HISTORY: u64 = 1000; -const CLIENT_DB_VER_STR: &'static str = "3"; +const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { /// Create a new client with given spec and DB path. @@ -209,34 +208,11 @@ impl Client { let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); - let mut opts = Options::new(); - opts.set_max_open_files(256); - opts.create_if_missing(true); - opts.set_use_fsync(false); - opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); - /* - opts.set_bytes_per_sync(8388608); - opts.set_disable_data_sync(false); - opts.set_block_cache_size_mb(1024); - opts.set_table_cache_num_shard_bits(6); - opts.set_max_write_buffer_number(32); - opts.set_write_buffer_size(536870912); - opts.set_target_file_size_base(1073741824); - opts.set_min_write_buffer_number_to_merge(4); - opts.set_level_zero_stop_writes_trigger(2000); - opts.set_level_zero_slowdown_writes_trigger(0); - opts.set_compaction_style(DBUniversalCompaction); - opts.set_max_background_compactions(4); - opts.set_max_background_flushes(4); - opts.set_filter_deletes(false); - opts.set_disable_auto_compactions(false);*/ - let mut state_path = path.to_path_buf(); state_path.push("state"); - let db = Arc::new(DB::open(&opts, state_path.to_str().unwrap()).unwrap()); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::new_with_arc(db.clone()); + let mut state_db = JournalDB::new(state_path.to_str().unwrap()); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/ethcore/src/extras.rs b/ethcore/src/extras.rs index 64d8afcb9..128357576 100644 --- a/ethcore/src/extras.rs +++ b/ethcore/src/extras.rs @@ -16,7 +16,6 @@ //! Blockchain DB extras. -use rocksdb::{DB, Writable}; use util::*; use header::BlockNumber; use receipt::Receipt; @@ -59,7 +58,7 @@ pub trait ExtrasReadable { K: ExtrasSliceConvertable; } -impl ExtrasWritable for W where W: Writable { +impl ExtrasWritable for DBTransaction { fn put_extras(&self, hash: &K, value: &T) where T: ExtrasIndexable + Encodable, K: ExtrasSliceConvertable { @@ -68,7 +67,7 @@ impl ExtrasWritable for W where W: Writable { } } -impl ExtrasReadable for DB { +impl ExtrasReadable for Database { fn get_extras(&self, hash: &K) -> Option where T: ExtrasIndexable + Decodable, K: ExtrasSliceConvertable { diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 54d0d1cd8..9c3fc6868 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -84,7 +84,6 @@ #[macro_use] extern crate ethcore_util as util; #[macro_use] extern crate lazy_static; extern crate rustc_serialize; -extern crate rocksdb; #[macro_use] extern crate heapsize; extern crate crypto; extern crate time; diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index dcffe8534..44ad667b9 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -19,7 +19,6 @@ use common::*; use spec::*; use blockchain::{BlockChain, BlockChainConfig}; use state::*; -use rocksdb::*; use evm::{Schedule, Factory}; use engine::*; use ethereum; @@ -226,8 +225,7 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { pub fn get_temp_journal_db() -> GuardedTempResult { let temp = RandomTempPath::new(); - let db = DB::open_default(temp.as_str()).unwrap(); - let journal_db = JournalDB::new(db); + let journal_db = JournalDB::new(temp.as_str()); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -244,8 +242,7 @@ pub fn get_temp_state() -> GuardedTempResult { } pub fn get_temp_journal_db_in(path: &Path) -> JournalDB { - let db = DB::open_default(path.to_str().unwrap()).unwrap(); - JournalDB::new(db) + JournalDB::new(path.to_str().unwrap()) } pub fn get_temp_state_in(path: &Path) -> State { diff --git a/util/Cargo.toml b/util/Cargo.toml index 6d2ebcd9b..96ad8a5c2 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -16,7 +16,7 @@ mio = "0.5.0" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" -rocksdb = "0.3" +rocksdb = { git = "https://github.com/arkpar/rust-rocksdb.git" } lazy_static = "0.1" eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } rust-crypto = "0.2.34" diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 7b810639b..cfcff6ea4 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -20,7 +20,7 @@ use common::*; use rlp::*; use hashdb::*; use memorydb::*; -use rocksdb::{DB, Writable, WriteBatch, IteratorMode}; +use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; @@ -33,7 +33,7 @@ use std::env; /// the removals actually take effect. pub struct JournalDB { overlay: MemoryDB, - backing: Arc, + backing: Arc, counters: Arc>>, } @@ -47,21 +47,25 @@ impl Clone for JournalDB { } } -const LATEST_ERA_KEY : [u8; 4] = [ b'l', b'a', b's', b't' ]; -const VERSION_KEY : [u8; 4] = [ b'j', b'v', b'e', b'r' ]; +// all keys must be at least 12 bytes +const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION: u32 = 2; +const DB_VERSION: u32 = 3; + +const PADDING : [u8; 10] = [ 0u8; 10 ]; impl JournalDB { - /// Create a new instance given a `backing` database. - pub fn new(backing: DB) -> JournalDB { - let db = Arc::new(backing); - JournalDB::new_with_arc(db) - } - /// Create a new instance given a shared `backing` database. - pub fn new_with_arc(backing: Arc) -> JournalDB { - if backing.iterator(IteratorMode::Start).next().is_some() { + /// Create a new instance from file + pub fn new(path: &str) -> JournalDB { + let opts = DatabaseConfig { + prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix + }; + let backing = Database::open(&opts, path).unwrap_or_else(|e| { + panic!("Error opening state db: {}", e); + }); + if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { Ok(Some(DB_VERSION)) => {}, v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) @@ -72,7 +76,7 @@ impl JournalDB { let counters = JournalDB::read_counters(&backing); JournalDB { overlay: MemoryDB::new(), - backing: backing, + backing: Arc::new(backing), counters: Arc::new(RwLock::new(counters)), } } @@ -82,7 +86,7 @@ impl JournalDB { pub fn new_temp() -> JournalDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + Self::new(dir.to_str().unwrap()) } /// Check if this database has any commits @@ -117,16 +121,17 @@ impl JournalDB { // and the key is safe to delete. // record new commit's details. - let batch = WriteBatch::new(); + let batch = DBTransaction::new(); let mut counters = self.counters.write().unwrap(); { let mut index = 0usize; let mut last; while try!(self.backing.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&now); r.append(&index); + r.append(&&PADDING[..]); last = r.drain(); &last })).is_some() { @@ -154,9 +159,10 @@ impl JournalDB { let mut to_remove: Vec = Vec::new(); let mut canon_inserts: Vec = Vec::new(); while let Some(rlp_data) = try!(self.backing.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&end_era); r.append(&index); + r.append(&&PADDING[..]); last = r.drain(); &last })) { @@ -226,16 +232,17 @@ impl JournalDB { self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } - fn read_counters(db: &DB) -> HashMap { + fn read_counters(db: &Database) -> HashMap { let mut res = HashMap::new(); if let Some(val) = db.get(&LATEST_ERA_KEY).expect("Low-level database error.") { let mut era = decode::(&val); loop { let mut index = 0usize; while let Some(rlp_data) = db.get({ - let mut r = RlpStream::new_list(2); + let mut r = RlpStream::new_list(3); r.append(&era); r.append(&index); + r.append(&&PADDING[..]); &r.drain() }).expect("Low-level database error.") { let rlp = Rlp::new(&rlp_data); @@ -259,7 +266,7 @@ impl JournalDB { impl HashDB for JournalDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iterator(IteratorMode::Start) { + for (key, _) in self.backing.iter() { let h = H256::from_slice(key.deref()); ret.insert(h, 1); } @@ -429,12 +436,11 @@ mod tests { #[test] fn reopen() { - use rocksdb::DB; let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let foo = { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -442,13 +448,13 @@ mod tests { }; { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); } { - let mut jdb = JournalDB::new(DB::open_default(dir.to_str().unwrap()).unwrap()); + let mut jdb = JournalDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); assert!(!jdb.exists(&foo)); diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs new file mode 100644 index 000000000..43a9fc532 --- /dev/null +++ b/util/src/kvdb.rs @@ -0,0 +1,206 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Key-Value store abstraction with RocksDB backend. + +use rocksdb::{DB, Writable, WriteBatch, IteratorMode, DBVector, DBIterator, + IndexType, Options, DBCompactionStyle, BlockBasedOptions, Direction}; + +/// Write transaction. Batches a sequence of put/delete operations for efficiency. +pub struct DBTransaction { + batch: WriteBatch, +} + +impl DBTransaction { + /// Create new transaction. + pub fn new() -> DBTransaction { + DBTransaction { batch: WriteBatch::new() } + } + + /// Insert a key-value pair in the transaction. Any existing value value will be overwritten upon write. + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { + self.batch.put(key, value) + } + + /// Delete value by key. + pub fn delete(&self, key: &[u8]) -> Result<(), String> { + self.batch.delete(key) + } +} + +/// Database configuration +pub struct DatabaseConfig { + /// Optional prefix size in bytes. Allows lookup by partial key. + pub prefix_size: Option +} + +/// Database iterator +pub struct DatabaseIterator<'a> { + iter: DBIterator<'a>, +} + +impl<'a> Iterator for DatabaseIterator<'a> { + type Item = (Box<[u8]>, Box<[u8]>); + + #[cfg_attr(feature="dev", allow(type_complexity))] + fn next(&mut self) -> Option<(Box<[u8]>, Box<[u8]>)> { + self.iter.next() + } +} + +/// Key-Value database. +pub struct Database { + db: DB, +} + +impl Database { + /// Open database with default settings. + pub fn open_default(path: &str) -> Result { + Database::open(&DatabaseConfig { prefix_size: None }, path) + } + + /// Open database file. Creates if it does not exist. + pub fn open(config: &DatabaseConfig, path: &str) -> Result { + let mut opts = Options::new(); + opts.set_max_open_files(256); + opts.create_if_missing(true); + opts.set_use_fsync(false); + opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); + /* + opts.set_bytes_per_sync(8388608); + opts.set_disable_data_sync(false); + opts.set_block_cache_size_mb(1024); + opts.set_table_cache_num_shard_bits(6); + opts.set_max_write_buffer_number(32); + opts.set_write_buffer_size(536870912); + opts.set_target_file_size_base(1073741824); + opts.set_min_write_buffer_number_to_merge(4); + opts.set_level_zero_stop_writes_trigger(2000); + opts.set_level_zero_slowdown_writes_trigger(0); + opts.set_compaction_style(DBUniversalCompaction); + opts.set_max_background_compactions(4); + opts.set_max_background_flushes(4); + opts.set_filter_deletes(false); + opts.set_disable_auto_compactions(false); + */ + + if let Some(size) = config.prefix_size { + let mut block_opts = BlockBasedOptions::new(); + block_opts.set_index_type(IndexType::HashSearch); + opts.set_block_based_table_factory(&block_opts); + opts.set_prefix_extractor_fixed_size(size); + } + let db = try!(DB::open(&opts, path)); + Ok(Database { db: db }) + } + + /// Insert a key-value pair in the transaction. Any existing value value will be overwritten. + pub fn put(&self, key: &[u8], value: &[u8]) -> Result<(), String> { + self.db.put(key, value) + } + + /// Delete value by key. + pub fn delete(&self, key: &[u8]) -> Result<(), String> { + self.db.delete(key) + } + + /// Commit transaction to database. + pub fn write(&self, tr: DBTransaction) -> Result<(), String> { + self.db.write(tr.batch) + } + + /// Get value by key. + pub fn get(&self, key: &[u8]) -> Result, String> { + self.db.get(key) + } + + /// Get value by partial key. Prefix size should match configured prefix size. + pub fn get_by_prefix(&self, prefix: &[u8]) -> Option> { + let mut iter = self.db.iterator(IteratorMode::From(prefix, Direction::forward)); + match iter.next() { + // TODO: use prefix_same_as_start read option (not availabele in C API currently) + Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, + _ => None + } + } + + /// Check if there is anything in the database. + pub fn is_empty(&self) -> bool { + self.db.iterator(IteratorMode::Start).next().is_none() + } + + /// Check if there is anything in the database. + pub fn iter(&self) -> DatabaseIterator { + DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) } + } +} + +#[cfg(test)] +mod tests { + use hash::*; + use super::*; + use devtools::*; + use std::str::FromStr; + use std::ops::Deref; + + fn test_db(config: &DatabaseConfig) { + let path = RandomTempPath::create_dir(); + let db = Database::open(config, path.as_path().to_str().unwrap()).unwrap(); + let key1 = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + let key2 = H256::from_str("03c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + let key3 = H256::from_str("01c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + + db.put(&key1, b"cat").unwrap(); + db.put(&key2, b"dog").unwrap(); + + assert_eq!(db.get(&key1).unwrap().unwrap().deref(), b"cat"); + + let contents: Vec<_> = db.iter().collect(); + assert_eq!(contents.len(), 2); + assert_eq!(&*contents[0].0, key1.deref()); + assert_eq!(&*contents[0].1, b"cat"); + assert_eq!(&*contents[1].0, key2.deref()); + assert_eq!(&*contents[1].1, b"dog"); + + db.delete(&key1).unwrap(); + assert!(db.get(&key1).unwrap().is_none()); + db.put(&key1, b"cat").unwrap(); + + let transaction = DBTransaction::new(); + transaction.put(&key3, b"elephant").unwrap(); + transaction.delete(&key1).unwrap(); + db.write(transaction).unwrap(); + assert!(db.get(&key1).unwrap().is_none()); + assert_eq!(db.get(&key3).unwrap().unwrap().deref(), b"elephant"); + + if config.prefix_size.is_some() { + assert_eq!(db.get_by_prefix(&key3).unwrap().deref(), b"elephant"); + assert_eq!(db.get_by_prefix(&key2).unwrap().deref(), b"dog"); + } + } + + #[test] + fn kvdb() { + let path = RandomTempPath::create_dir(); + let smoke = Database::open_default(path.as_path().to_str().unwrap()).unwrap(); + assert!(smoke.is_empty()); + test_db(&DatabaseConfig { prefix_size: None }); + test_db(&DatabaseConfig { prefix_size: Some(1) }); + test_db(&DatabaseConfig { prefix_size: Some(8) }); + test_db(&DatabaseConfig { prefix_size: Some(32) }); + } +} + diff --git a/util/src/lib.rs b/util/src/lib.rs index 1fba48e00..0d074df8c 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -129,6 +129,7 @@ pub mod hashdb; pub mod memorydb; pub mod overlaydb; pub mod journaldb; +pub mod kvdb; mod math; pub mod crypto; pub mod triehash; @@ -161,4 +162,5 @@ pub use semantic_version::*; pub use network::*; pub use io::*; pub use log::*; +pub use kvdb::*; diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index f8e9c3eee..f4ed2d5d6 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -26,7 +26,7 @@ use std::ops::*; use std::sync::*; use std::env; use std::collections::HashMap; -use rocksdb::{DB, Writable, IteratorMode}; +use kvdb::{Database}; /// Implementation of the HashDB trait for a disk-backed database with a memory overlay. /// @@ -38,15 +38,15 @@ use rocksdb::{DB, Writable, IteratorMode}; /// queries have an immediate effect in terms of these functions. pub struct OverlayDB { overlay: MemoryDB, - backing: Arc, + backing: Arc, } impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - pub fn new(backing: DB) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) } + pub fn new(backing: Database) -> OverlayDB { Self::new_with_arc(Arc::new(backing)) } /// Create a new instance of OverlayDB given a `backing` database. - pub fn new_with_arc(backing: Arc) -> OverlayDB { + pub fn new_with_arc(backing: Arc) -> OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: backing } } @@ -54,7 +54,7 @@ impl OverlayDB { pub fn new_temp() -> OverlayDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + Self::new(Database::open_default(dir.to_str().unwrap()).unwrap()) } /// Commit all memory operations to the backing database. @@ -164,7 +164,7 @@ impl OverlayDB { impl HashDB for OverlayDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iterator(IteratorMode::Start) { + for (key, _) in self.backing.iter() { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; ret.insert(h, r as i32); @@ -318,7 +318,7 @@ fn overlaydb_complex() { fn playpen() { use std::fs; { - let db: DB = DB::open_default("/tmp/test").unwrap(); + let db: Database = Database::open_default("/tmp/test").unwrap(); db.put(b"test", b"test2").unwrap(); match db.get(b"test") { Ok(Some(value)) => println!("Got value {:?}", value.deref()),