Merge pull request #6720 from paritytech/kvdb_split

separated kvdb into 3 crates: kvdb, kvdb-memorydb && kvdb-rocksdb

ethcore/Cargo.toml

@@ -56,6 +56,8 @@ rand = "0.3"
 rlp = { path = "../util/rlp" }
 rlp_derive = { path = "../util/rlp_derive" }
 kvdb = { path = "../util/kvdb" }
+kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
+kvdb-memorydb = { path = "../util/kvdb-memorydb" }
 util-error = { path = "../util/error" }
 snappy = { path = "../util/snappy" }
 migration = { path = "../util/migration" }

ethcore/light/Cargo.toml

@@ -40,6 +40,8 @@ stats = { path = "../../util/stats" }
 hash = { path = "../../util/hash" }
 triehash = { path = "../../util/triehash" }
 kvdb = { path = "../../util/kvdb" }
+kvdb-rocksdb = { path = "../../util/kvdb-rocksdb" }
+kvdb-memorydb = { path = "../../util/kvdb-memorydb" }
 
 [features]
 default = []

ethcore/light/src/client/header_chain.rs

@@ -728,13 +728,14 @@ mod tests {
 	use ethcore::header::Header;
 	use ethcore::spec::Spec;
   	use cache::Cache;
-	use kvdb::{in_memory, KeyValueDB};
+	use kvdb::KeyValueDB;
+	use kvdb_memorydb;
 
 	use time::Duration;
 	use parking_lot::Mutex;
 
 	fn make_db() -> Arc<KeyValueDB> {
-		Arc::new(in_memory(0))
+		Arc::new(kvdb_memorydb::create(0))
 	}
 
 	#[test]

ethcore/light/src/client/mod.rs

@@ -36,7 +36,8 @@ use bigint::prelude::U256;
 use bigint::hash::H256;
 use futures::{IntoFuture, Future};
 
-use kvdb::{KeyValueDB, CompactionProfile};
+use kvdb::KeyValueDB;
+use kvdb_rocksdb::CompactionProfile;
 
 use self::fetch::ChainDataFetcher;
 use self::header_chain::{AncestryIter, HeaderChain};
@@ -214,7 +215,7 @@ impl<T: ChainDataFetcher> Client<T> {
 		io_channel: IoChannel<ClientIoMessage>,
 		cache: Arc<Mutex<Cache>>
 	) -> Self {
-		let db = ::kvdb::in_memory(0);
+		let db = ::kvdb_memorydb::create(0);
 
 		Client::new(
 			config,

ethcore/light/src/client/service.rs

@@ -25,7 +25,7 @@ use ethcore::db;
 use ethcore::service::ClientIoMessage;
 use ethcore::spec::Spec;
 use io::{IoContext, IoError, IoHandler, IoService};
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 
 use cache::Cache;
 use parking_lot::Mutex;

ethcore/light/src/lib.rs

@@ -92,6 +92,8 @@ extern crate vm;
 extern crate hash;
 extern crate triehash;
 extern crate kvdb;
+extern crate kvdb_memorydb;
+extern crate kvdb_rocksdb;
 
 #[cfg(feature = "ipc")]
 extern crate ethcore_ipc as ipc;

ethcore/node_filter/Cargo.toml

@@ -11,10 +11,12 @@ ethcore = { path = ".."}
 ethcore-util = { path = "../../util" }
 ethcore-bigint = { path = "../../util/bigint" }
 ethcore-bytes = { path = "../../util/bytes" }
-ethcore-io = { path = "../../util/io" }
 ethcore-network = { path = "../../util/network" }
-kvdb = { path = "../../util/kvdb" }
 native-contracts = { path = "../native_contracts" }
 futures = "0.1"
 log = "0.3"
 parking_lot = "0.4"
+
+[dev-dependencies]
+kvdb-memorydb = { path = "../../util/kvdb-memorydb" }
+ethcore-io = { path = "../../util/io" }

ethcore/node_filter/src/lib.rs

@@ -24,9 +24,14 @@ extern crate ethcore_network as network;
 extern crate native_contracts;
 extern crate futures;
 extern crate parking_lot;
-extern crate kvdb;
-#[cfg(test)] extern crate ethcore_io as io;
-#[macro_use] extern crate log;
+
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+extern crate kvdb_memorydb;
+#[cfg(test)]
+extern crate ethcore_io as io;
 
 use std::sync::Weak;
 use std::collections::HashMap;
@@ -135,7 +140,7 @@ mod test {
 		let contract_addr = Address::from_str("0000000000000000000000000000000000000005").unwrap();
 		let data = include_bytes!("../res/node_filter.json");
 		let spec = Spec::load(&::std::env::temp_dir(), &data[..]).unwrap();
-		let client_db = Arc::new(::kvdb::in_memory(::ethcore::db::NUM_COLUMNS.unwrap_or(0)));
+		let client_db = Arc::new(::kvdb_memorydb::create(::ethcore::db::NUM_COLUMNS.unwrap_or(0)));
 
 		let client = Client::new(
 			ClientConfig::default(),

ethcore/src/blockchain/blockchain.rs

@@ -1479,7 +1479,8 @@ mod tests {
 	use std::sync::Arc;
 	use rustc_hex::FromHex;
 	use hash::keccak;
-	use kvdb::{in_memory, KeyValueDB};
+	use kvdb::KeyValueDB;
+	use kvdb_memorydb;
 	use bigint::hash::*;
 	use receipt::{Receipt, TransactionOutcome};
 	use blockchain::{BlockProvider, BlockChain, Config, ImportRoute};
@@ -1493,7 +1494,7 @@ mod tests {
 	use header::BlockNumber;
 
 	fn new_db() -> Arc<KeyValueDB> {
-		Arc::new(in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+		Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)))
 	}
 
 	fn new_chain(genesis: &[u8], db: Arc<KeyValueDB>) -> BlockChain {

ethcore/src/client/config.rs

@@ -21,7 +21,7 @@ use std::fmt::{Display, Formatter, Error as FmtError};
 use mode::Mode as IpcMode;
 use verification::{VerifierType, QueueConfig};
 use util::journaldb;
-use kvdb::CompactionProfile;
+use kvdb_rocksdb::CompactionProfile;
 
 pub use std::time::Duration;
 pub use blockchain::Config as BlockChainConfig;

ethcore/src/client/evm_test_client.rs

@@ -21,8 +21,7 @@ use std::sync::Arc;
 use bigint::prelude::U256;
 use bigint::hash::H256;
 use util::journaldb;
-use trie;
-use bytes;
+use {trie, kvdb_memorydb, bytes};
 use kvdb::{self, KeyValueDB};
 use {state, state_db, client, executive, trace, transaction, db, spec, pod_state};
 use factory::Factories;
@@ -128,7 +127,7 @@ impl<'a> EvmTestClient<'a> {
 	}
 
 	fn state_from_spec(spec: &'a spec::Spec, factories: &Factories) -> Result<state::State<state_db::StateDB>, EvmTestError> {
-		let db = Arc::new(kvdb::in_memory(db::NUM_COLUMNS.expect("We use column-based DB; qed")));
+		let db = Arc::new(kvdb_memorydb::create(db::NUM_COLUMNS.expect("We use column-based DB; qed")));
 		let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, db::COL_STATE);
 		let mut state_db = state_db::StateDB::new(journal_db, 5 * 1024 * 1024);
 		state_db = spec.ensure_db_good(state_db, factories)?;
@@ -150,7 +149,7 @@ impl<'a> EvmTestClient<'a> {
 	}
 
 	fn state_from_pod(spec: &'a spec::Spec, factories: &Factories, pod_state: pod_state::PodState) -> Result<state::State<state_db::StateDB>, EvmTestError> {
-		let db = Arc::new(kvdb::in_memory(db::NUM_COLUMNS.expect("We use column-based DB; qed")));
+		let db = Arc::new(kvdb_memorydb::create(db::NUM_COLUMNS.expect("We use column-based DB; qed")));
 		let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, db::COL_STATE);
 		let state_db = state_db::StateDB::new(journal_db, 5 * 1024 * 1024);
 		let mut state = state::State::new(

ethcore/src/client/test_client.rs

@@ -27,7 +27,7 @@ use bigint::prelude::U256;
 use bigint::hash::H256;
 use parking_lot::RwLock;
 use util::*;
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use bytes::Bytes;
 use rlp::*;
 use ethkey::{Generator, Random};

ethcore/src/json_tests/chain.rs

@@ -57,7 +57,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 			};
 
 			{
-				let db = Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+				let db = Arc::new(::kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 				let mut config = ClientConfig::default();
 				config.history = 8;
 				let client = Client::new(

ethcore/src/lib.rs

@@ -113,6 +113,8 @@ extern crate ansi_term;
 extern crate semantic_version;
 extern crate unexpected;
 extern crate kvdb;
+extern crate kvdb_rocksdb;
+extern crate kvdb_memorydb;
 extern crate util_error;
 extern crate snappy;
 extern crate migration;

ethcore/src/migrations/state/v7.rs

@@ -22,7 +22,7 @@ use std::collections::HashMap;
 use bigint::hash::H256;
 use util::Address;
 use bytes::Bytes;
-use kvdb::Database;
+use kvdb_rocksdb::Database;
 use migration::{Batch, Config, Error, Migration, SimpleMigration, Progress};
 use hash::keccak;
 use std::sync::Arc;

ethcore/src/migrations/v10.rs

@@ -26,7 +26,8 @@ use migration::{Error, Migration, Progress, Batch, Config};
 use util::journaldb;
 use bigint::hash::H256;
 use trie::Trie;
-use kvdb::{Database, DBTransaction};
+use kvdb::DBTransaction;
+use kvdb_rocksdb::Database;
 
 /// Account bloom upgrade routine. If bloom already present, does nothing.
 /// If database empty (no best block), does nothing.

ethcore/src/migrations/v9.rs

@@ -18,7 +18,7 @@
 //! This migration consolidates all databases into single one using Column Families.
 
 use rlp::{Rlp, RlpStream};
-use kvdb::Database;
+use kvdb_rocksdb::Database;
 use migration::{Batch, Config, Error, Migration, Progress};
 use std::sync::Arc;
 

ethcore/src/service.rs

@@ -19,7 +19,8 @@
 use std::sync::Arc;
 use std::path::Path;
 use bigint::hash::H256;
-use kvdb::{Database, DatabaseConfig, KeyValueDB};
+use kvdb::KeyValueDB;
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use bytes::Bytes;
 use io::*;
 use spec::Spec;

ethcore/src/snapshot/service.rs

@@ -40,7 +40,7 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard};
 use util_error::UtilError;
 use bytes::Bytes;
 use util::journaldb::Algorithm;
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use snappy;
 
 /// Helper for removing directories in case of error.
@@ -682,7 +682,7 @@ mod tests {
 	#[test]
 	fn cannot_finish_with_invalid_chunks() {
 		use bigint::hash::H256;
-		use kvdb::DatabaseConfig;
+		use kvdb_rocksdb::DatabaseConfig;
 
 		let spec = get_test_spec();
 		let dir = RandomTempPath::new();

ethcore/src/snapshot/tests/proof_of_authority.rs

@@ -31,7 +31,7 @@ use tests::helpers;
 use transaction::{Transaction, Action, SignedTransaction};
 
 use util::Address;
-use kvdb;
+use kvdb_memorydb;
 
 const PASS: &'static str = "";
 const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated.
@@ -238,7 +238,7 @@ fn fixed_to_contract_only() {
 	assert_eq!(client.chain_info().best_block_number, 11);
 	let reader = snapshot_helpers::snap(&*client);
 
-	let new_db = kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0));
+	let new_db = kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0));
 	let spec = spec_fixed_to_contract();
 
 	// ensure fresh engine's step matches.
@@ -270,7 +270,7 @@ fn fixed_to_contract_to_contract() {
 
 	assert_eq!(client.chain_info().best_block_number, 16);
 	let reader = snapshot_helpers::snap(&*client);
-	let new_db = kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0));
+	let new_db = kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0));
 	let spec = spec_fixed_to_contract();
 
 	for _ in 0..16 { spec.engine.step() }

ethcore/src/snapshot/tests/proof_of_work.rs

@@ -26,7 +26,8 @@ use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
 
 use parking_lot::Mutex;
 use snappy;
-use kvdb::{self, KeyValueDB, DBTransaction};
+use kvdb::{KeyValueDB, DBTransaction};
+use kvdb_memorydb;
 
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
@@ -43,7 +44,7 @@ fn chunk_and_restore(amount: u64) {
 	let mut snapshot_path = new_path.as_path().to_owned();
 	snapshot_path.push("SNAP");
 
-	let old_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+	let old_db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 	let bc = BlockChain::new(Default::default(), &genesis, old_db.clone());
 
 	// build the blockchain.
@@ -80,7 +81,7 @@ fn chunk_and_restore(amount: u64) {
 	writer.into_inner().finish(manifest.clone()).unwrap();
 
 	// restore it.
-	let new_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+	let new_db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 	let new_chain = BlockChain::new(Default::default(), &genesis, new_db.clone());
 	let mut rebuilder = SNAPSHOT_MODE.rebuilder(new_chain, new_db.clone(), &manifest).unwrap();
 
@@ -127,7 +128,7 @@ fn checks_flag() {
 
 	let chunk = stream.out();
 
-	let db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+	let db = Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 	let engine = ::spec::Spec::new_test().engine;
 	let chain = BlockChain::new(Default::default(), &genesis, db.clone());
 

ethcore/src/snapshot/tests/service.rs

@@ -27,7 +27,7 @@ use tests::helpers::generate_dummy_client_with_spec_and_data;
 
 use devtools::RandomTempPath;
 use io::IoChannel;
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 
 struct NoopDBRestore;
 

ethcore/src/snapshot/tests/state.rs

@@ -27,7 +27,7 @@ use error::Error;
 use rand::{XorShiftRng, SeedableRng};
 use bigint::hash::H256;
 use util::journaldb::{self, Algorithm};
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use memorydb::MemoryDB;
 use parking_lot::Mutex;
 use devtools::RandomTempPath;

ethcore/src/spec/spec.rs

@@ -672,13 +672,13 @@ impl Spec {
 	pub fn genesis_epoch_data(&self) -> Result<Vec<u8>, String> {
 		use transaction::{Action, Transaction};
 		use util::journaldb;
-		use kvdb;
+		use kvdb_memorydb;
 
 		let genesis = self.genesis_header();
 
 		let factories = Default::default();
 		let mut db = journaldb::new(
-			Arc::new(kvdb::in_memory(0)),
+			Arc::new(kvdb_memorydb::create(0)),
 			journaldb::Algorithm::Archive,
 			None,
 		);

ethcore/src/tests/client.rs

@@ -27,7 +27,7 @@ use tests::helpers::*;
 use types::filter::Filter;
 use bigint::prelude::U256;
 use util::*;
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use devtools::*;
 use miner::Miner;
 use spec::Spec;

ethcore/src/tests/helpers.rs

@@ -232,7 +232,7 @@ pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> Arc<Client> {
 }
 
 fn new_db() -> Arc<::kvdb::KeyValueDB> {
-	Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+	Arc::new(::kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)))
 }
 
 pub fn generate_dummy_blockchain(block_number: u32) -> BlockChain {

ethcore/src/tests/trace.rs

@@ -27,7 +27,7 @@ use client::*;
 use tests::helpers::*;
 use devtools::RandomTempPath;
 use client::{BlockChainClient, Client, ClientConfig};
-use kvdb::{Database, DatabaseConfig};
+use kvdb_rocksdb::{Database, DatabaseConfig};
 use std::sync::Arc;
 use header::Header;
 use miner::Miner;

ethcore/src/trace/db.rs

@@ -416,7 +416,8 @@ mod tests {
 	use bigint::prelude::U256;
 	use bigint::hash::H256;
 	use util::Address;
-	use kvdb::{DBTransaction, in_memory, KeyValueDB};
+	use kvdb::{DBTransaction, KeyValueDB};
+	use kvdb_memorydb;
 	use header::BlockNumber;
 	use trace::{Config, TraceDB, Database as TraceDatabase, DatabaseExtras, ImportRequest};
 	use trace::{Filter, LocalizedTrace, AddressesFilter, TraceError};
@@ -467,7 +468,7 @@ mod tests {
 	}
 
 	fn new_db() -> Arc<KeyValueDB> {
-		Arc::new(in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+		Arc::new(kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)))
 	}
 
 	#[test]

ethcore/src/tx_filter.rs

@@ -178,7 +178,7 @@ mod test {
 		"#;
 
 		let spec = Spec::load(&::std::env::temp_dir(), spec_data.as_bytes()).unwrap();
-		let client_db = Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+		let client_db = Arc::new(::kvdb_memorydb::create(::db::NUM_COLUMNS.unwrap_or(0)));
 
 		let client = Client::new(
 			ClientConfig::default(),