Extract state-db from ethcore (#10858)

* WIP move errors, pod_account and state account to own crates

* Sort out dependencies, fix broken code and tests
Remove botched ethcore-error crate

* remove template line

* fix review feedback

* Remove test-only AccountDBMut::new

* Extract AccountDB to account-db

* Move Substate to state-account – wip

* Add lib.rs

* cleanup

* test failure

* test failure 2

* third time's the charm

* Add factories crate

* Use new factories crate

* Use factories crate

* Extract trace

* Fix tests

* Sort out parity-util-mem and parking_lot

* cleanup

* WIP port over the rest of state from ethcore

* Collect all impls for Machine

* some notes

* Rename pod-account to pod

* Move PodState to pod crate

* Use PodState from pod crate

* Fix use clause for json tests

* Sort out evmbin

* Add missing code and use PodState

* Move code that depends on Machine and Executive to own module

* Sort out cloning errors, fix ethcore to use new state crate

* Do without funky From impls

* Fix ethcore tests

* Fixes around the project to use new state crate

* Add back the more specific impls of StateOrBlock From conversions

* Move execute to freestanding function and remove it from trait
Sort out the error handling in executive_state by moving the result types from state to ethcore
Undo the verbose code added to work around the StateOrBlock From conversions

* cleanup

* Fix "error: enum variants on type aliases are experimental"

* Bring back the state tests
Fix whitespace

* remove ethcore/state/mod.rs

* cleanup

* cleanup

* Cleanup state-account errors

* Fix more todos
Add module docs

* Add error.rs

* Fixup Cargo.lock

* Smaller ethcore API is fine

* Add `to-pod-full` feature to state-account
Fix evmbin

* Fix a few more test failures

* Fix RPC test build

* Baptize the new trait

* Remove resolved TODOs

* Rename state-account to account-state

* Do not re-export the trace crate

* Don't export state_db from ethcore

* Let private-tx use StateDB. :(

* Remove ethcore/src/pod_state.rs

* Inner type does not need to be pub/pub(crate)

* optimise imports

* Revert "Inner type does not need to be pub/pub(crate)"

This reverts commit 2f839f8a0f72f71334da64620f57e6dd6039f06b.

* Move DatabaseExtras to ethcore-blockchain

* Add database_extra module to ethcore-blockchain

* Remove to-pod-full feature

* cosmetics

* New crate: state-db

* Add new crate

* Sort out the merge

* Fix unclean merge

* Add license

Cargo.lock

@@ -914,6 +914,7 @@ dependencies = [
  "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "state-db 0.1.0",
  "stats 0.1.0",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "time-utils 0.1.0",
@@ -1219,6 +1220,7 @@ dependencies = [
  "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "state-db 0.1.0",
  "time-utils 0.1.0",
  "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "trace 0.1.0",
@@ -3935,6 +3937,28 @@ name = "stable_deref_trait"
 version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "state-db"
+version = "0.1.0"
+dependencies = [
+ "account-state 0.1.0",
+ "common-types 0.1.0",
+ "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ethcore 1.12.0",
+ "ethcore-bloom-journal 0.1.0",
+ "ethcore-db 0.1.0",
+ "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "journaldb 0.2.0",
+ "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keccak-hasher 0.1.1",
+ "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memory-cache 0.1.0",
+ "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "static_assertions"
 version = "0.2.5"

ethcore/Cargo.toml

@@ -63,6 +63,7 @@ serde = "1.0"
 serde_derive = "1.0"
 account-state = { path = "account-state" }
 stats = { path = "../util/stats" }
+state-db = { path = "state-db" }
 tempdir = { version = "0.3", optional = true }
 time-utils = { path = "../util/time-utils" }
 trace = { path = "trace" }

ethcore/private-tx/Cargo.toml

@@ -6,6 +6,7 @@ license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
+account-state = { path = "../account-state" }
 common-types = { path = "../types" }
 derive_more = "0.14.0"
 ethabi = "8.0"
@@ -35,7 +36,7 @@ rustc-hex = "1.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-account-state = { path = "../account-state" }
+state-db = { path = "../state-db" }
 time-utils = { path = "../../util/time-utils" }
 tiny-keccak = "1.4"
 trace = { path = "../trace" }

ethcore/private-tx/src/lib.rs

@@ -23,6 +23,7 @@ mod messages;
 mod error;
 mod log;
 
+extern crate account_state;
 extern crate common_types as types;
 extern crate ethabi;
 extern crate ethcore;
@@ -46,8 +47,8 @@ extern crate rlp;
 extern crate serde_derive;
 extern crate serde;
 extern crate serde_json;
-extern crate account_state;
 extern crate rustc_hex;
+extern crate state_db;
 extern crate trace;
 extern crate transaction_pool as txpool;
 extern crate url;
@@ -94,7 +95,7 @@ use ethcore::client::{
 	Call, BlockInfo
 };
 use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache};
-use ethcore::StateDB;
+use state_db::StateDB;
 use account_state::State;
 use trace::{Tracer, VMTracer};
 use call_contract::CallContract;

ethcore/service/src/service.rs

@@ -119,8 +119,8 @@ impl ClientService {
 		let snapshot_params = SnapServiceParams {
 			engine: spec.engine.clone(),
 			genesis_block: spec.genesis_block(),
-			restoration_db_handler: restoration_db_handler,
+			restoration_db_handler,
-			pruning: pruning,
+			pruning,
 			channel: io_service.channel(),
 			snapshot_root: snapshot_path.into(),
 			client: client.clone(),

ethcore/src/lib.rs

@@ -54,6 +54,7 @@
 //!   ```
 
 extern crate account_db;
+extern crate account_state;
 extern crate ansi_term;
 extern crate common_types as types;
 extern crate crossbeam_utils;
@@ -97,8 +98,8 @@ extern crate parity_util_mem;
 extern crate parity_util_mem as malloc_size_of;
 extern crate rustc_hex;
 extern crate serde;
+extern crate state_db;
 extern crate stats;
-extern crate account_state;
 extern crate time_utils;
 extern crate trace;
 extern crate triehash_ethereum as triehash;
@@ -163,7 +164,6 @@ pub mod spec;
 pub mod verification;
 
 mod externalities;
-mod state_db;
 mod transaction_ext;
 mod tx_filter;
 
@@ -177,4 +177,3 @@ pub mod test_helpers;
 pub use executive::contract_address;
 pub use evm::CreateContractAddress;
 pub use trie::TrieSpec;
-pub use state_db::StateDB;

ethcore/src/machine.rs

@@ -83,9 +83,9 @@ impl Machine {
 	pub fn regular(params: CommonParams, builtins: BTreeMap<Address, Builtin>) -> Machine {
 		let tx_filter = TransactionFilter::from_params(&params).map(Arc::new);
 		Machine {
-			params: params,
+			params,
 			builtins: Arc::new(builtins),
-			tx_filter: tx_filter,
+			tx_filter,
 			ethash_extensions: None,
 			schedule_rules: None,
 		}

ethcore/src/spec/spec.rs

@@ -543,8 +543,8 @@ fn load_from(spec_params: SpecParams, s: ethjson::spec::Spec) -> Result<Spec, Er
 		gas_used: g.gas_used,
 		timestamp: g.timestamp,
 		extra_data: g.extra_data,
-		seal_rlp: seal_rlp,
+		seal_rlp,
-		hardcoded_sync: hardcoded_sync,
+		hardcoded_sync,
 		constructors: s.accounts
 			.constructors()
 			.into_iter()

ethcore/state-db/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+description = "State database"
+name = "state-db"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+license = "GPL-3.0"
+edition = "2018"
+
+[dependencies]
+account-state = { path = "../account-state" }
+bloom_journal = { package = "ethcore-bloom-journal", path = "../../util/bloom" }
+common-types = { path = "../types"}
+ethcore-db = { path = "../db" }
+ethereum-types = "0.6.0"
+hash-db = "0.12.4"
+keccak-hash = "0.2.0"
+keccak-hasher = { path = "../../util/keccak-hasher" }
+journaldb = { path = "../../util/journaldb" }
+kvdb = "0.1.0"
+log = "0.4.6"
+lru-cache = "0.1.2"
+memory-cache = { path = "../../util/memory-cache" }
+parking_lot = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.5"
+# Used for test helpers
+ethcore = { path = "..", features = ["test-helpers"] }

ethcore/state-db/src/lib.rs

@@ -16,24 +16,25 @@
 
 //! State database abstraction. For more info, see the doc for `StateDB`
 
-use std::collections::{VecDeque, HashSet};
+use std::collections::{HashSet, VecDeque};
 use std::io;
 use std::sync::Arc;
 
-use bloom_journal::{Bloom, BloomJournal};
+use ethereum_types::{Address, H256};
-use db::COL_ACCOUNT_BLOOM;
-use ethereum_types::{H256, Address};
-use hash::keccak;
 use hash_db::HashDB;
-use journaldb::JournalDB;
+use keccak_hash::keccak;
-use keccak_hasher::KeccakHasher;
+use kvdb::{DBTransaction, DBValue, KeyValueDB};
-use kvdb::{KeyValueDB, DBTransaction, DBValue};
+use log::trace;
 use lru_cache::LruCache;
-use memory_cache::MemoryLruCache;
 use parking_lot::Mutex;
-use types::BlockNumber;
 
 use account_state::{self, Account};
+use bloom_journal::{Bloom, BloomJournal};
+use common_types::BlockNumber;
+use ethcore_db::COL_ACCOUNT_BLOOM;
+use journaldb::JournalDB;
+use keccak_hasher::KeccakHasher;
+use memory_cache::MemoryLruCache;
 
 /// Value used to initialize bloom bitmap size.
 ///
@@ -68,7 +69,7 @@ struct AccountCache {
 struct CacheQueueItem {
 	/// Account address.
 	address: Address,
-	/// Acccount data or `None` if account does not exist.
+	/// Account data or `None` if account does not exist.
 	account: SyncAccount,
 	/// Indicates that the account was modified before being
 	/// added to the cache.
@@ -143,7 +144,7 @@ impl StateDB {
 		let cache_items = acc_cache_size / ::std::mem::size_of::<Option<Account>>();
 
 		StateDB {
-			db: db,
+			db,
 			account_cache: Arc::new(Mutex::new(AccountCache {
 				accounts: LruCache::new(cache_items),
 				modifications: VecDeque::new(),
@@ -151,7 +152,7 @@ impl StateDB {
 			code_cache: Arc::new(Mutex::new(MemoryLruCache::new(code_cache_size))),
 			local_cache: Vec::new(),
 			account_bloom: Arc::new(Mutex::new(bloom)),
-			cache_size: cache_size,
+			cache_size,
 			parent_hash: None,
 			commit_hash: None,
 			commit_number: None,
@@ -159,7 +160,7 @@ impl StateDB {
 	}
 
 	/// Loads accounts bloom from the database
-	/// This bloom is used to handle request for the non-existant account fast
+	/// This bloom is used to handle request for the non-existent account fast
 	pub fn load_bloom(db: &dyn KeyValueDB) -> Bloom {
 		let hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY)
 			.expect("Low-level database error");
@@ -177,7 +178,7 @@ impl StateDB {
 			let key: [u8; 8] = (i as u64).to_le_bytes();
 			bloom_parts[i] = db.get(COL_ACCOUNT_BLOOM, &key).expect("low-level database error")
 				.map(|val| {
-					assert!(val.len() == 8, "low-level database error");
+					assert_eq!(val.len(), 8, "low-level database error");
 					let mut buff = [0u8; 8];
 					buff.copy_from_slice(&*val);
 					u64::from_le_bytes(buff)
@@ -233,7 +234,7 @@ impl StateDB {
 		let cache = &mut *cache;
 
 		// Purge changes from re-enacted and retracted blocks.
-		// Filter out commiting block if any.
+		// Filter out committing block if any.
 		let mut clear = false;
 		for block in enacted.iter().filter(|h| self.commit_hash.as_ref().map_or(true, |p| *h != p)) {
 			clear = clear || {
@@ -419,11 +420,11 @@ impl account_state::Backend for StateDB {
 		self.db.as_hash_db_mut()
 	}
 
-	fn add_to_account_cache(&mut self, addr: Address, data: Option<Account>, modified: bool) {
+	fn add_to_account_cache(&mut self, address: Address, data: Option<Account>, modified: bool) {
 		self.local_cache.push(CacheQueueItem {
-			address: addr,
+			address,
 			account: SyncAccount(data),
-			modified: modified,
+			modified,
 		})
 	}
 
@@ -484,10 +485,11 @@ unsafe impl Sync for SyncAccount {}
 
 #[cfg(test)]
 mod tests {
-	use ethereum_types::{H256, U256, Address};
+	use ethereum_types::{Address, H256, U256};
 	use kvdb::DBTransaction;
-	use test_helpers::get_temp_state_db;
+
 	use account_state::{Account, Backend};
+	use ethcore::test_helpers::get_temp_state_db;
 
 	#[test]
 	fn state_db_smoke() {