Separate migrations from util (#6690)

* separate migration from util and make its dependencies into libs:

* snappy
* kvdb
* error
* common

* renamed common -> macros

* util error does not depend on snappy module

* ethsync does not depend on util nor ethcore_error

* nibbleslice and nibblevec merged with patricia_trie crate

* removed unused dependencies from util

* util journaldb traits does not need to be public

* util_error

* fixed ethcore compile error

* ignore .swo files

* Update chain.rs

ethcore/src/blockchain/blockchain.rs

@@ -26,7 +26,6 @@ use bigint::prelude::U256;
 use bigint::hash::{H256, H2048};
 use parking_lot::{Mutex, RwLock};
 use bytes::Bytes;
-use util::*;
 use rlp::*;
 use header::*;
 use super::extras::*;
@@ -47,6 +46,7 @@ use encoded;
 use engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition};
 use rayon::prelude::*;
 use ansi_term::Colour;
+use kvdb::{DBTransaction, KeyValueDB};
 
 const LOG_BLOOMS_LEVELS: usize = 3;
 const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16;
@@ -1479,7 +1479,7 @@ mod tests {
 	use std::sync::Arc;
 	use rustc_hex::FromHex;
 	use hash::keccak;
-	use util::kvdb::KeyValueDB;
+	use kvdb::{in_memory, KeyValueDB};
 	use bigint::hash::*;
 	use receipt::{Receipt, TransactionOutcome};
 	use blockchain::{BlockProvider, BlockChain, Config, ImportRoute};
@@ -1493,7 +1493,7 @@ mod tests {
 	use header::BlockNumber;
 
 	fn new_db() -> Arc<KeyValueDB> {
-		Arc::new(::util::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+		Arc::new(in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
 	}
 
 	fn new_chain(genesis: &[u8], db: Arc<KeyValueDB>) -> BlockChain {

ethcore/src/blockchain/extras.rs

@@ -28,7 +28,7 @@ use receipt::Receipt;
 use heapsize::HeapSizeOf;
 use bigint::prelude::U256;
 use bigint::hash::{H256, H264};
-use util::kvdb::PREFIX_LEN as DB_PREFIX_LEN;
+use kvdb::PREFIX_LEN as DB_PREFIX_LEN;
 
 /// Represents index of extra data in database
 #[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)]

ethcore/src/client/client.rs

@@ -26,10 +26,10 @@ use itertools::Itertools;
 use hash::keccak;
 use timer::PerfTimer;
 use bytes::Bytes;
-use util::{journaldb, DBValue};
-use util::{Address, UtilError};
+use util::{Address, journaldb, DBValue};
+use util_error::UtilError;
 use trie::{TrieSpec, TrieFactory, Trie};
-use util::kvdb::*;
+use kvdb::*;
 
 // other
 use bigint::prelude::U256;
@@ -2090,7 +2090,7 @@ mod tests {
 		use std::time::Duration;
 		use std::sync::Arc;
 		use std::sync::atomic::{AtomicBool, Ordering};
-		use util::kvdb::DBTransaction;
+		use kvdb::DBTransaction;
 
 		let client = generate_dummy_client(0);
 		let genesis = client.chain_info().best_block_hash;

ethcore/src/client/config.rs

@@ -20,7 +20,8 @@ use std::fmt::{Display, Formatter, Error as FmtError};
 
 use mode::Mode as IpcMode;
 use verification::{VerifierType, QueueConfig};
-use util::{journaldb, CompactionProfile};
+use util::journaldb;
+use kvdb::CompactionProfile;
 
 pub use std::time::Duration;
 pub use blockchain::Config as BlockChainConfig;

ethcore/src/client/error.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use util::UtilError;
+use util_error::UtilError;
 use std::fmt::{Display, Formatter, Error as FmtError};
 
 use trie::TrieError;

ethcore/src/client/evm_test_client.rs

@@ -23,7 +23,7 @@ use bigint::hash::H256;
 use util::journaldb;
 use trie;
 use bytes;
-use util::kvdb::{self, KeyValueDB};
+use kvdb::{self, KeyValueDB};
 use {state, state_db, client, executive, trace, transaction, db, spec, pod_state};
 use factory::Factories;
 use evm::{self, VMType, FinalizationResult};

ethcore/src/client/test_client.rs

@@ -27,6 +27,7 @@ use bigint::prelude::U256;
 use bigint::hash::H256;
 use parking_lot::RwLock;
 use util::*;
+use kvdb::{Database, DatabaseConfig};
 use bytes::Bytes;
 use rlp::*;
 use ethkey::{Generator, Random};

ethcore/src/db.rs

@@ -20,7 +20,7 @@ use std::ops::Deref;
 use std::hash::Hash;
 use std::collections::HashMap;
 use parking_lot::RwLock;
-use util::{DBTransaction, KeyValueDB};
+use kvdb::{DBTransaction, KeyValueDB};
 
 use rlp;
 

ethcore/src/error.rs

@@ -17,9 +17,12 @@
 //! General error types for use in ethcore.
 
 use std::fmt;
+use kvdb;
 use bigint::prelude::U256;
 use bigint::hash::H256;
 use util::*;
+use util_error::UtilError;
+use snappy::InvalidInput;
 use unexpected::{Mismatch, OutOfBounds};
 use trie::TrieError;
 use io::*;
@@ -299,6 +302,8 @@ impl From<Error> for TransactionImportError {
 pub enum Error {
 	/// Client configuration error.
 	Client(ClientError),
+	/// Database error.
+	Database(kvdb::Error),
 	/// Error concerning a utility.
 	Util(UtilError),
 	/// Error concerning block processing.
@@ -322,7 +327,7 @@ pub enum Error {
 	/// Standard io error.
 	StdIo(::std::io::Error),
 	/// Snappy error.
-	Snappy(::util::snappy::InvalidInput),
+	Snappy(InvalidInput),
 	/// Snapshot error.
 	Snapshot(SnapshotError),
 	/// Consensus vote error.
@@ -337,6 +342,7 @@ impl fmt::Display for Error {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		match *self {
 			Error::Client(ref err) => err.fmt(f),
+			Error::Database(ref err) => err.fmt(f),
 			Error::Util(ref err) => err.fmt(f),
 			Error::Io(ref err) => err.fmt(f),
 			Error::Block(ref err) => err.fmt(f),
@@ -370,6 +376,12 @@ impl From<ClientError> for Error {
 	}
 }
 
+impl From<kvdb::Error> for Error {
+	fn from(err: kvdb::Error) -> Error {
+		Error::Database(err)
+	}
+}
+
 impl From<TransactionError> for Error {
 	fn from(err: TransactionError) -> Error {
 		Error::Transaction(err)
@@ -434,8 +446,8 @@ impl From<BlockImportError> for Error {
 	}
 }
 
-impl From<snappy::InvalidInput> for Error {
-	fn from(err: snappy::InvalidInput) -> Error {
+impl From<::snappy::InvalidInput> for Error {
+	fn from(err: ::snappy::InvalidInput) -> Error {
 		Error::Snappy(err)
 	}
 }

ethcore/src/json_tests/chain.rs

@@ -57,7 +57,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 			};
 
 			{
-				let db = Arc::new(::util::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+				let db = Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
 				let client = Client::new(
 					ClientConfig::default(),
 					&spec,

ethcore/src/lib.rs

@@ -112,6 +112,10 @@ extern crate triehash;
 extern crate ansi_term;
 extern crate semantic_version;
 extern crate unexpected;
+extern crate kvdb;
+extern crate util_error;
+extern crate snappy;
+extern crate migration;
 
 #[macro_use]
 extern crate rlp_derive;
@@ -124,11 +128,12 @@ extern crate table;
 extern crate bloomable;
 extern crate vm;
 extern crate wasm;
+extern crate ethcore_util as util;
 
 #[macro_use]
-extern crate log;
+extern crate macros;
 #[macro_use]
-extern crate ethcore_util as util;
+extern crate log;
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]

ethcore/src/migrations/blocks/v8.rs

@@ -16,7 +16,7 @@
 
 //! This migration compresses the state db.
 
-use util::migration::{SimpleMigration, Progress};
+use migration::{SimpleMigration, Progress};
 use rlp::{Compressible, UntrustedRlp, RlpType};
 
 /// Compressing migration.

ethcore/src/migrations/extras/v6.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use util::migration::SimpleMigration;
+use migration::SimpleMigration;
 
 /// This migration reduces the sizes of keys and moves `ExtrasIndex` byte from back to the front.
 pub struct ToV6;

ethcore/src/migrations/mod.rs

@@ -16,7 +16,7 @@
 
 //! Database migrations.
 
-use util::migration::ChangeColumns;
+use migration::ChangeColumns;
 
 pub mod state;
 pub mod blocks;

ethcore/src/migrations/state/v7.rs

@@ -22,8 +22,8 @@ use std::collections::HashMap;
 use bigint::hash::H256;
 use util::Address;
 use bytes::Bytes;
-use util::kvdb::Database;
-use util::migration::{Batch, Config, Error, Migration, SimpleMigration, Progress};
+use kvdb::Database;
+use migration::{Batch, Config, Error, Migration, SimpleMigration, Progress};
 use hash::keccak;
 use std::sync::Arc;
 

ethcore/src/migrations/v10.rs

@@ -22,11 +22,11 @@ use state_db::{ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET, StateDB};
 use trie::TrieDB;
 use views::HeaderView;
 use bloom_journal::Bloom;
-use util::migration::{Error, Migration, Progress, Batch, Config};
+use migration::{Error, Migration, Progress, Batch, Config};
 use util::journaldb;
 use bigint::hash::H256;
 use trie::Trie;
-use util::{Database, DBTransaction};
+use kvdb::{Database, DBTransaction};
 
 /// Account bloom upgrade routine. If bloom already present, does nothing.
 /// If database empty (no best block), does nothing.

ethcore/src/migrations/v9.rs

@@ -18,8 +18,8 @@
 //! This migration consolidates all databases into single one using Column Families.
 
 use rlp::{Rlp, RlpStream};
-use util::kvdb::Database;
-use util::migration::{Batch, Config, Error, Migration, Progress};
+use kvdb::Database;
+use migration::{Batch, Config, Error, Migration, Progress};
 use std::sync::Arc;
 
 /// Which part of block to preserve

ethcore/src/service.rs

@@ -19,7 +19,7 @@
 use std::sync::Arc;
 use std::path::Path;
 use bigint::hash::H256;
-use util::*;
+use kvdb::{Database, DatabaseConfig, KeyValueDB};
 use bytes::Bytes;
 use io::*;
 use spec::Spec;

ethcore/src/snapshot/consensus/authority.rs

@@ -35,7 +35,7 @@ use snapshot::{Error, ManifestData};
 use itertools::{Position, Itertools};
 use rlp::{RlpStream, UntrustedRlp};
 use bigint::hash::H256;
-use util::KeyValueDB;
+use kvdb::KeyValueDB;
 use bytes::Bytes;
 
 /// Snapshot creation and restoration for PoA chains.

ethcore/src/snapshot/consensus/mod.rs

@@ -25,7 +25,7 @@ use engines::EthEngine;
 use snapshot::{Error, ManifestData};
 
 use bigint::hash::H256;
-use util::kvdb::KeyValueDB;
+use kvdb::KeyValueDB;
 
 mod authority;
 mod work;

ethcore/src/snapshot/consensus/work.rs

@@ -31,7 +31,7 @@ use engines::EthEngine;
 use snapshot::{Error, ManifestData};
 use snapshot::block::AbridgedBlock;
 use bigint::hash::H256;
-use util::KeyValueDB;
+use kvdb::KeyValueDB;
 use bytes::Bytes;
 use rlp::{RlpStream, UntrustedRlp};
 use rand::OsRng;

ethcore/src/snapshot/mod.rs

@@ -32,11 +32,12 @@ use ids::BlockId;
 
 use bigint::prelude::U256;
 use bigint::hash::H256;
-use util::{HashDB, DBValue, snappy};
+use util::{HashDB, DBValue};
+use snappy;
 use bytes::Bytes;
 use parking_lot::Mutex;
 use util::journaldb::{self, Algorithm, JournalDB};
-use util::kvdb::KeyValueDB;
+use kvdb::KeyValueDB;
 use trie::{TrieDB, TrieDBMut, Trie, TrieMut};
 use rlp::{RlpStream, UntrustedRlp};
 use bloom_journal::Bloom;

ethcore/src/snapshot/service.rs

@@ -37,11 +37,11 @@ use io::IoChannel;
 
 use bigint::hash::H256;
 use parking_lot::{Mutex, RwLock, RwLockReadGuard};
-use util::UtilError;
+use util_error::UtilError;
 use bytes::Bytes;
 use util::journaldb::Algorithm;
-use util::kvdb::{Database, DatabaseConfig};
-use util::snappy;
+use kvdb::{Database, DatabaseConfig};
+use snappy;
 
 /// Helper for removing directories in case of error.
 struct Guard(bool, PathBuf);
@@ -682,7 +682,7 @@ mod tests {
 	#[test]
 	fn cannot_finish_with_invalid_chunks() {
 		use bigint::hash::H256;
-		use util::kvdb::DatabaseConfig;
+		use kvdb::DatabaseConfig;
 
 		let spec = get_test_spec();
 		let dir = RandomTempPath::new();

ethcore/src/snapshot/tests/helpers.rs

@@ -31,7 +31,8 @@ use snapshot::io::{SnapshotReader, PackedWriter, PackedReader};
 use devtools::{RandomTempPath, GuardedTempResult};
 use rand::Rng;
 
-use util::{DBValue, KeyValueDB};
+use util::DBValue;
+use kvdb::KeyValueDB;
 use bigint::hash::H256;
 use hashdb::HashDB;
 use util::journaldb;
@@ -165,7 +166,7 @@ pub fn restore(
 	genesis: &[u8],
 ) -> Result<(), ::error::Error> {
 	use std::sync::atomic::AtomicBool;
-	use util::snappy;
+	use snappy;
 
 	let flag = AtomicBool::new(true);
 	let components = engine.snapshot_components().unwrap();

ethcore/src/snapshot/tests/proof_of_authority.rs

@@ -31,7 +31,7 @@ use tests::helpers;
 use transaction::{Transaction, Action, SignedTransaction};
 
 use util::Address;
-use util::kvdb;
+use kvdb;
 
 const PASS: &'static str = "";
 const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated.

ethcore/src/snapshot/tests/proof_of_work.rs

@@ -25,8 +25,8 @@ use snapshot::{chunk_secondary, Error as SnapshotError, Progress, SnapshotCompon
 use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
 
 use parking_lot::Mutex;
-use util::snappy;
-use util::kvdb::{self, KeyValueDB, DBTransaction};
+use snappy;
+use kvdb::{self, KeyValueDB, DBTransaction};
 
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;

ethcore/src/snapshot/tests/service.rs

@@ -27,7 +27,7 @@ use tests::helpers::generate_dummy_client_with_spec_and_data;
 
 use devtools::RandomTempPath;
 use io::IoChannel;
-use util::kvdb::{Database, DatabaseConfig};
+use kvdb::{Database, DatabaseConfig};
 
 struct NoopDBRestore;
 

ethcore/src/snapshot/tests/state.rs

@@ -27,7 +27,7 @@ use error::Error;
 use rand::{XorShiftRng, SeedableRng};
 use bigint::hash::H256;
 use util::journaldb::{self, Algorithm};
-use util::kvdb::{Database, DatabaseConfig};
+use kvdb::{Database, DatabaseConfig};
 use memorydb::MemoryDB;
 use parking_lot::Mutex;
 use devtools::RandomTempPath;
@@ -76,7 +76,7 @@ fn snap_and_restore() {
 
 		for chunk_hash in &reader.manifest().state_hashes {
 			let raw = reader.chunk(*chunk_hash).unwrap();
-			let chunk = ::util::snappy::decompress(&raw).unwrap();
+			let chunk = ::snappy::decompress(&raw).unwrap();
 
 			rebuilder.feed(&chunk, &flag).unwrap();
 		}
@@ -190,7 +190,7 @@ fn checks_flag() {
 
 		for chunk_hash in &reader.manifest().state_hashes {
 			let raw = reader.chunk(*chunk_hash).unwrap();
-			let chunk = ::util::snappy::decompress(&raw).unwrap();
+			let chunk = ::snappy::decompress(&raw).unwrap();
 
 			match rebuilder.feed(&chunk, &flag) {
 				Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {},

ethcore/src/spec/spec.rs

@@ -667,7 +667,8 @@ impl Spec {
 	/// constructor.
 	pub fn genesis_epoch_data(&self) -> Result<Vec<u8>, String> {
 		use transaction::{Action, Transaction};
-		use util::{journaldb, kvdb};
+		use util::journaldb;
+		use kvdb;
 
 		let genesis = self.genesis_header();
 

ethcore/src/state_db.rs

@@ -19,14 +19,15 @@ use std::sync::Arc;
 use lru_cache::LruCache;
 use util::cache::MemoryLruCache;
 use util::journaldb::JournalDB;
-use util::kvdb::KeyValueDB;
+use kvdb::{KeyValueDB, DBTransaction};
 use bigint::hash::H256;
 use hashdb::HashDB;
 use state::{self, Account};
 use header::BlockNumber;
 use hash::keccak;
 use parking_lot::Mutex;
-use util::{Address, DBTransaction, UtilError};
+use util::Address;
+use util_error::UtilError;
 use bloom_journal::{Bloom, BloomJournal};
 use db::COL_ACCOUNT_BLOOM;
 use byteorder::{LittleEndian, ByteOrder};
@@ -460,7 +461,8 @@ impl state::Backend for StateDB {
 mod tests {
 	use bigint::prelude::U256;
 	use bigint::hash::H256;
-	use util::{Address, DBTransaction};
+	use util::Address;
+	use kvdb::DBTransaction;
 	use tests::helpers::*;
 	use state::{Account, Backend};
 	use ethcore_logger::init_log;

ethcore/src/tests/client.rs

@@ -27,6 +27,7 @@ use tests::helpers::*;
 use types::filter::Filter;
 use bigint::prelude::U256;
 use util::*;
+use kvdb::{Database, DatabaseConfig};
 use devtools::*;
 use miner::Miner;
 use spec::Spec;

ethcore/src/tests/helpers.rs

@@ -231,8 +231,8 @@ pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> Arc<Client> {
 	client
 }
 
-fn new_db() -> Arc<KeyValueDB> {
-	Arc::new(::util::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+fn new_db() -> Arc<::kvdb::KeyValueDB> {
+	Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
 }
 
 pub fn generate_dummy_blockchain(block_number: u32) -> BlockChain {

ethcore/src/tests/trace.rs

@@ -27,7 +27,7 @@ use client::*;
 use tests::helpers::*;
 use devtools::RandomTempPath;
 use client::{BlockChainClient, Client, ClientConfig};
-use util::kvdb::{Database, DatabaseConfig};
+use kvdb::{Database, DatabaseConfig};
 use std::sync::Arc;
 use header::Header;
 use miner::Miner;

ethcore/src/trace/db.rs

@@ -22,7 +22,7 @@ use bloomchain::{Number, Config as BloomConfig};
 use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
 use heapsize::HeapSizeOf;
 use bigint::hash::{H256, H264};
-use util::{KeyValueDB, DBTransaction};
+use kvdb::{KeyValueDB, DBTransaction};
 use parking_lot::RwLock;
 use header::BlockNumber;
 use trace::{LocalizedTrace, Config, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras};
@@ -415,7 +415,8 @@ mod tests {
 	use std::sync::Arc;
 	use bigint::prelude::U256;
 	use bigint::hash::H256;
-	use util::{Address, DBTransaction};
+	use util::Address;
+	use kvdb::{DBTransaction, in_memory, KeyValueDB};
 	use header::BlockNumber;
 	use trace::{Config, TraceDB, Database as TraceDatabase, DatabaseExtras, ImportRequest};
 	use trace::{Filter, LocalizedTrace, AddressesFilter, TraceError};
@@ -465,8 +466,8 @@ mod tests {
 		}
 	}
 
-	fn new_db() -> Arc<::util::kvdb::KeyValueDB> {
-		Arc::new(::util::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
+	fn new_db() -> Arc<KeyValueDB> {
+		Arc::new(in_memory(::db::NUM_COLUMNS.unwrap_or(0)))
 	}
 
 	#[test]

ethcore/src/trace/mod.rs

@@ -39,7 +39,8 @@ pub use self::types::filter::{Filter, AddressesFilter};
 
 use bigint::prelude::U256;
 use bigint::hash::H256;
-use util::{Address, DBTransaction};
+use util::Address;
+use kvdb::DBTransaction;
 use bytes::Bytes;
 use self::trace::{Call, Create};
 use vm::ActionParams;

ethcore/src/tx_filter.rs

@@ -178,7 +178,7 @@ mod test {
 		"#;
 
 		let spec = Spec::load(&::std::env::temp_dir(), spec_data.as_bytes()).unwrap();
-		let client_db = Arc::new(::util::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
+		let client_db = Arc::new(::kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
 
 		let client = Client::new(
 			ClientConfig::default(),