Separate migrations from util (#6690)

* separate migration from util and make its dependencies into libs:

* snappy
* kvdb
* error
* common

* renamed common -> macros

* util error does not depend on snappy module

* ethsync does not depend on util nor ethcore_error

* nibbleslice and nibblevec merged with patricia_trie crate

* removed unused dependencies from util

* util journaldb traits does not need to be public

* util_error

* fixed ethcore compile error

* ignore .swo files

* Update chain.rs

util/Cargo.toml

@@ -24,18 +24,16 @@ target_info = "0.1"
 ethcore-bigint = { path = "bigint", features = ["heapsizeof"] }
 parking_lot = "0.4"
 tiny-keccak= "1.0"
-ethcore-bloom-journal = { path = "bloom" }
-regex = "0.2"
 lru-cache = "0.1.0"
 ethcore-logger = { path = "../logger" }
 triehash = { path = "triehash" }
 error-chain = "0.11.0-rc.2"
 hashdb = { path = "hashdb" }
 patricia_trie = { path = "patricia_trie" }
-nibbleslice = { path = "nibbleslice" }
-nibblevec = { path = "nibblevec" }
 ethcore-bytes = { path = "bytes" }
 memorydb = { path = "memorydb" }
+util-error = { path = "error" }
+kvdb = { path = "kvdb" }
 
 [features]
 default = []

util/error/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "util-error"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+rlp = { path = "../rlp" }
+ethcore-bigint = { path = "../bigint" }
+error-chain = "0.11.0-rc.2"
+rustc-hex = "1.0"

util/error/src/lib.rs

@@ -19,7 +19,14 @@
 #![allow(missing_docs)]
 #![allow(unknown_lints)]
 
-use std::{self, fmt};
+#[macro_use]
+extern crate error_chain;
+
+extern crate ethcore_bigint as bigint;
+extern crate rlp;
+extern crate rustc_hex;
+
+use std::fmt;
 use rustc_hex::FromHexError;
 use rlp::DecoderError;
 use bigint::hash::H256;
@@ -59,7 +66,6 @@ error_chain! {
 		Io(::std::io::Error);
 		FromHex(FromHexError);
 		Decoder(DecoderError);
-		Snappy(::snappy::InvalidInput);
 		BaseData(BaseDataError);
 	}
 }

util/kvdb/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "kvdb"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+log = "0.3"
+ethcore-bytes = { path = "../bytes" }
+ethcore-bigint = { path = "../bigint" }
+ethcore-devtools = { path = "../../devtools" }
+elastic-array = "0.9"
+hashdb = { path = "../hashdb" }
+parking_lot = "0.4"
+regex = "0.2"
+rlp = { path = "../rlp" }
+rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" }
+error-chain = "0.11.0-rc.2"

util/kvdb/src/lib.rs

@@ -16,9 +16,23 @@
 
 //! Key-Value store abstraction with `RocksDB` backend.
 
-use std::{mem, fs};
+#[macro_use]
+extern crate log;
+#[macro_use]
+extern crate error_chain;
+
+extern crate ethcore_bytes as bytes;
+extern crate ethcore_bigint as bigint;
+extern crate ethcore_devtools as devtools;
+extern crate elastic_array;
+extern crate hashdb;
+extern crate parking_lot;
+extern crate rlp;
+extern crate rocksdb;
+extern crate regex;
+
+use std::{mem, fs, io};
 use std::collections::{HashMap, BTreeMap};
-use std::io::ErrorKind;
 use std::marker::PhantomData;
 use std::path::{PathBuf, Path};
 use parking_lot::{Mutex, MutexGuard, RwLock};
@@ -28,10 +42,8 @@ use hashdb::DBValue;
 use rlp::{UntrustedRlp, RlpType, Compressible};
 use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
 	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column, ReadOptions};
-use UtilError;
 use bytes::Bytes;
 
-
 #[cfg(target_os = "linux")]
 use regex::Regex;
 #[cfg(target_os = "linux")]
@@ -46,6 +58,16 @@ const DB_WRITE_BUFFER_SIZE: usize = 2048 * 1000;
 /// Required length of prefixes.
 pub const PREFIX_LEN: usize = 12;
 
+error_chain! {
+	types {
+		Error, ErrorKind, ResultExt;
+	}
+
+	foreign_links {
+		Io(io::Error);
+	}
+}
+
 /// Write transaction. Batches a sequence of put/delete operations for efficiency.
 #[derive(Default, Clone, PartialEq)]
 pub struct DBTransaction {
@@ -182,7 +204,7 @@ pub trait KeyValueDB: Sync + Send {
 		-> Box<Iterator<Item=(Box<[u8]>, Box<[u8]>)> + 'a>;
 
 	/// Attempt to replace this database with a new one located at the given path.
-	fn restore(&self, new_db: &str) -> Result<(), UtilError>;
+	fn restore(&self, new_db: &str) -> Result<(), Error>;
 }
 
 /// A key-value database fulfilling the `KeyValueDB` trait, living in memory.
@@ -279,7 +301,7 @@ impl KeyValueDB for InMemory {
 		}
 	}
 
-	fn restore(&self, _new_db: &str) -> Result<(), UtilError> {
+	fn restore(&self, _new_db: &str) -> Result<(), Error> {
 		Err("Attempted to restore in-memory database".into())
 	}
 }
@@ -777,7 +799,7 @@ impl Database {
 	}
 
 	/// Restore the database from a copy at given path.
-	pub fn restore(&self, new_db: &str) -> Result<(), UtilError> {
+	pub fn restore(&self, new_db: &str) -> Result<(), Error> {
 		self.close();
 
 		let mut backup_db = PathBuf::from(&self.path);
@@ -786,7 +808,7 @@ impl Database {
 
 		let existed = match fs::rename(&self.path, &backup_db) {
 			Ok(_) => true,
-			Err(e) => if let ErrorKind::NotFound = e.kind() {
+			Err(e) => if let io::ErrorKind::NotFound = e.kind() {
 				false
 			} else {
 				return Err(e.into());
@@ -889,7 +911,7 @@ impl KeyValueDB for Database {
 		Box::new(unboxed.into_iter().flat_map(|inner| inner))
 	}
 
-	fn restore(&self, new_db: &str) -> Result<(), UtilError> {
+	fn restore(&self, new_db: &str) -> Result<(), Error> {
 		Database::restore(self, new_db)
 	}
 }

util/macros/Cargo.toml

@@ -1,7 +1,6 @@
 [package]
-name = "nibbleslice"
+name = "macros"
 version = "0.1.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
-elastic-array = "0.9"

util/macros/src/lib.rs

@@ -82,8 +82,8 @@ macro_rules! map_into {
 
 #[macro_export]
 macro_rules! flush {
-	($arg:expr) => ($crate::common::flush($arg.into()));
-	($($arg:tt)*) => ($crate::common::flush(format!("{}", format_args!($($arg)*))));
+	($arg:expr) => ($crate::flush($arg.into()));
+	($($arg:tt)*) => ($crate::flush(format!("{}", format_args!($($arg)*))));
 }
 
 #[macro_export]

util/migration/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "migration"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+log = "0.3"
+macros = { path = "../macros" }
+kvdb = { path = "../kvdb" }
+ethcore-devtools = { path = "../../devtools" }

util/migration/src/lib.rs

@@ -18,13 +18,21 @@
 #[cfg(test)]
 mod tests;
 
+#[macro_use]
+extern crate log;
+#[macro_use]
+extern crate macros;
+
+extern crate ethcore_devtools as devtools;
+extern crate kvdb;
+
 use std::collections::BTreeMap;
 use std::fs;
 use std::fmt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction};
+use kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction};
 
 /// Migration config.
 #[derive(Clone)]

util/migration/src/tests.rs

@@ -21,7 +21,7 @@
 use std::collections::BTreeMap;
 use std::sync::Arc;
 use std::path::{Path, PathBuf};
-use migration::{Batch, Config, Error, SimpleMigration, Migration, Manager};
+use {Batch, Config, Error, SimpleMigration, Migration, Manager, ChangeColumns};
 use kvdb::Database;
 use devtools::RandomTempPath;
 
@@ -232,7 +232,7 @@ fn change_columns() {
 	use kvdb::DatabaseConfig;
 
 	let mut manager = Manager::new(Config::default());
-	manager.add_migration(::migration::ChangeColumns {
+	manager.add_migration(ChangeColumns {
 		pre_columns: None,
 		post_columns: Some(4),
 		version: 1,

util/patricia_trie/Cargo.toml

@@ -12,8 +12,6 @@ ethcore-bigint = { path = "../bigint" }
 hash = { path = "../hash" }
 hashdb = { path = "../hashdb" }
 rlp = { path = "../rlp" }
-nibbleslice = { path = "../nibbleslice" }
-nibblevec = { path = "../nibblevec" }
 triehash = { path = "../triehash" }
 memorydb = { path = "../memorydb" }
 ethcore-logger = { path = "../../logger" }

util/patricia_trie/src/lib.rs

@@ -21,8 +21,6 @@ extern crate hash as keccak;
 extern crate rlp;
 extern crate hashdb;
 extern crate ethcore_bytes as bytes;
-extern crate nibbleslice;
-extern crate nibblevec;
 extern crate elastic_array;
 extern crate memorydb;
 extern crate ethcore_logger;
@@ -54,6 +52,8 @@ pub mod recorder;
 mod fatdb;
 mod fatdbmut;
 mod lookup;
+mod nibbleslice;
+mod nibblevec;
 
 pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
 pub use self::triedbmut::TrieDBMut;

util/patricia_trie/src/nibbleslice.rs

@@ -15,7 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets.
-extern crate elastic_array;
 
 use std::cmp::*;
 use std::fmt;
@@ -26,9 +25,8 @@ use elastic_array::ElasticArray36;
 /// This is an immutable struct. No operations actually change it.
 ///
 /// # Example
-/// ```rust
-/// extern crate nibbleslice;
-/// use nibbleslice::*;
+/// ```snippet
+/// use patricia_trie::nibbleslice::NibbleSlice;
 /// fn main() {
 ///   let d1 = &[0x01u8, 0x23, 0x45];
 ///   let d2 = &[0x34u8, 0x50, 0x12];

util/patricia_trie/src/nibblevec.rs

@@ -14,13 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-
 //! An owning, nibble-oriented byte vector.
-extern crate nibbleslice;
-extern crate elastic_array;
-
-use nibbleslice::NibbleSlice;
 use elastic_array::ElasticArray36;
+use nibbleslice::NibbleSlice;
 
 /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`.
 #[derive(Clone, PartialEq, Eq, Debug)]

util/patricia_trie/src/node.rs

@@ -15,7 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use elastic_array::ElasticArray36;
-use nibbleslice::*;
+use nibbleslice::NibbleSlice;
 use nibblevec::NibbleVec;
 use bytes::*;
 use rlp::*;

util/patricia_trie/src/triedb.rs

@@ -16,7 +16,7 @@
 
 use std::fmt;
 use hashdb::*;
-use nibbleslice::*;
+use nibbleslice::NibbleSlice;
 use rlp::*;
 use super::node::{Node, OwnedNode};
 use super::lookup::Lookup;

util/snappy/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
-name = "nibblevec"
+name = "snappy"
 version = "0.1.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
-elastic-array = "0.9"
-nibbleslice = { path = "../nibbleslice" }
+libc = "0.2.7"

util/snappy/src/lib.rs

@@ -15,8 +15,9 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Snappy compression bindings.
+extern crate libc;
 
-use std::{self, fmt};
+use std::fmt;
 use libc::{c_char, c_int, size_t};
 
 const SNAPPY_OK: c_int = 0;

util/src/journaldb/mod.rs

@@ -20,7 +20,7 @@ use std::{fmt, str};
 use std::sync::Arc;
 
 /// Export the journaldb module.
-pub mod traits;
+mod traits;
 mod archivedb;
 mod earlymergedb;
 mod overlayrecentdb;

util/src/journaldb/refcounteddb.rs

@@ -27,7 +27,7 @@ use super::{DB_PREFIX_LEN, LATEST_ERA_KEY};
 use super::traits::JournalDB;
 use kvdb::{KeyValueDB, DBTransaction};
 use bigint::hash::H256;
-use UtilError;
+use error::UtilError;
 use bytes::Bytes;
 
 /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay

util/src/journaldb/traits.rs

@@ -20,7 +20,7 @@ use std::sync::Arc;
 use hashdb::*;
 use kvdb::{self, DBTransaction};
 use bigint::hash::H256;
-use UtilError;
+use error::UtilError;
 use bytes::Bytes;
 
 /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually

util/src/lib.rs

@@ -100,7 +100,6 @@ extern crate ethcore_bytes as bytes;
 extern crate parking_lot;
 extern crate tiny_keccak;
 extern crate rlp;
-extern crate regex;
 extern crate lru_cache;
 extern crate heapsize;
 extern crate ethcore_logger;
@@ -108,22 +107,15 @@ extern crate hash as keccak;
 extern crate hashdb;
 extern crate memorydb;
 extern crate patricia_trie as trie;
-
-#[macro_use]
-extern crate error_chain;
+extern crate kvdb;
+extern crate util_error as error;
 
 #[macro_use]
 extern crate log as rlog;
 
-#[macro_use]
-pub mod common;
-pub mod error;
 pub mod misc;
-pub mod migration;
 pub mod overlaydb;
 pub mod journaldb;
-pub mod kvdb;
-pub mod snappy;
 pub mod cache;
 
 pub use misc::*;
@@ -131,8 +123,6 @@ pub use hashdb::*;
 pub use memorydb::MemoryDB;
 pub use overlaydb::*;
 pub use journaldb::JournalDB;
-pub use kvdb::*;
-pub use error::UtilError;
 
 /// 160-bit integer representing account address
 pub type Address = bigint::hash::H160;