Extract state-db from ethcore (#10858)

* WIP move errors, pod_account and state account to own crates

* Sort out dependencies, fix broken code and tests
Remove botched ethcore-error crate

* remove template line

* fix review feedback

* Remove test-only AccountDBMut::new

* Extract AccountDB to account-db

* Move Substate to state-account – wip

* Add lib.rs

* cleanup

* test failure

* test failure 2

* third time's the charm

* Add factories crate

* Use new factories crate

* Use factories crate

* Extract trace

* Fix tests

* Sort out parity-util-mem and parking_lot

* cleanup

* WIP port over the rest of state from ethcore

* Collect all impls for Machine

* some notes

* Rename pod-account to pod

* Move PodState to pod crate

* Use PodState from pod crate

* Fix use clause for json tests

* Sort out evmbin

* Add missing code and use PodState

* Move code that depends on Machine and Executive to own module

* Sort out cloning errors, fix ethcore to use new state crate

* Do without funky From impls

* Fix ethcore tests

* Fixes around the project to use new state crate

* Add back the more specific impls of StateOrBlock From conversions

* Move execute to freestanding function and remove it from trait
Sort out the error handling in executive_state by moving the result types from state to ethcore
Undo the verbose code added to work around the StateOrBlock From conversions

* cleanup

* Fix "error: enum variants on type aliases are experimental"

* Bring back the state tests
Fix whitespace

* remove ethcore/state/mod.rs

* cleanup

* cleanup

* Cleanup state-account errors

* Fix more todos
Add module docs

* Add error.rs

* Fixup Cargo.lock

* Smaller ethcore API is fine

* Add `to-pod-full` feature to state-account
Fix evmbin

* Fix a few more test failures

* Fix RPC test build

* Baptize the new trait

* Remove resolved TODOs

* Rename state-account to account-state

* Do not re-export the trace crate

* Don't export state_db from ethcore

* Let private-tx use StateDB. :(

* Remove ethcore/src/pod_state.rs

* Inner type does not need to be pub/pub(crate)

* optimise imports

* Revert "Inner type does not need to be pub/pub(crate)"

This reverts commit 2f839f8a0f72f71334da64620f57e6dd6039f06b.

* Move DatabaseExtras to ethcore-blockchain

* Add database_extra module to ethcore-blockchain

* Remove to-pod-full feature

* cosmetics

* New crate: state-db

* Add new crate

* Sort out the merge

* Fix unclean merge

* Add license
This commit is contained in:
David 2019-07-08 23:01:47 +02:00 committed by Seun LanLege
parent 073d242d1e
commit 0f337171bf
10 changed files with 89 additions and 33 deletions

24
Cargo.lock generated
View File

@ -914,6 +914,7 @@ dependencies = [
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
"state-db 0.1.0",
"stats 0.1.0", "stats 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"time-utils 0.1.0", "time-utils 0.1.0",
@ -1219,6 +1220,7 @@ dependencies = [
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
"state-db 0.1.0",
"time-utils 0.1.0", "time-utils 0.1.0",
"tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trace 0.1.0", "trace 0.1.0",
@ -3935,6 +3937,28 @@ name = "stable_deref_trait"
version = "1.1.1" version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "state-db"
version = "0.1.0"
dependencies = [
"account-state 0.1.0",
"common-types 0.1.0",
"env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
"ethcore 1.12.0",
"ethcore-bloom-journal 0.1.0",
"ethcore-db 0.1.0",
"ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)",
"journaldb 0.2.0",
"keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"keccak-hasher 0.1.1",
"kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"memory-cache 0.1.0",
"parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "static_assertions" name = "static_assertions"
version = "0.2.5" version = "0.2.5"

View File

@ -63,6 +63,7 @@ serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
account-state = { path = "account-state" } account-state = { path = "account-state" }
stats = { path = "../util/stats" } stats = { path = "../util/stats" }
state-db = { path = "state-db" }
tempdir = { version = "0.3", optional = true } tempdir = { version = "0.3", optional = true }
time-utils = { path = "../util/time-utils" } time-utils = { path = "../util/time-utils" }
trace = { path = "trace" } trace = { path = "trace" }

View File

@ -6,6 +6,7 @@ license = "GPL-3.0"
authors = ["Parity Technologies <admin@parity.io>"] authors = ["Parity Technologies <admin@parity.io>"]
[dependencies] [dependencies]
account-state = { path = "../account-state" }
common-types = { path = "../types" } common-types = { path = "../types" }
derive_more = "0.14.0" derive_more = "0.14.0"
ethabi = "8.0" ethabi = "8.0"
@ -35,7 +36,7 @@ rustc-hex = "1.0"
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
serde_json = "1.0" serde_json = "1.0"
account-state = { path = "../account-state" } state-db = { path = "../state-db" }
time-utils = { path = "../../util/time-utils" } time-utils = { path = "../../util/time-utils" }
tiny-keccak = "1.4" tiny-keccak = "1.4"
trace = { path = "../trace" } trace = { path = "../trace" }

View File

@ -23,6 +23,7 @@ mod messages;
mod error; mod error;
mod log; mod log;
extern crate account_state;
extern crate common_types as types; extern crate common_types as types;
extern crate ethabi; extern crate ethabi;
extern crate ethcore; extern crate ethcore;
@ -46,8 +47,8 @@ extern crate rlp;
extern crate serde_derive; extern crate serde_derive;
extern crate serde; extern crate serde;
extern crate serde_json; extern crate serde_json;
extern crate account_state;
extern crate rustc_hex; extern crate rustc_hex;
extern crate state_db;
extern crate trace; extern crate trace;
extern crate transaction_pool as txpool; extern crate transaction_pool as txpool;
extern crate url; extern crate url;
@ -94,7 +95,7 @@ use ethcore::client::{
Call, BlockInfo Call, BlockInfo
}; };
use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache}; use ethcore::miner::{self, Miner, MinerService, pool_client::NonceCache};
use ethcore::StateDB; use state_db::StateDB;
use account_state::State; use account_state::State;
use trace::{Tracer, VMTracer}; use trace::{Tracer, VMTracer};
use call_contract::CallContract; use call_contract::CallContract;

View File

@ -119,8 +119,8 @@ impl ClientService {
let snapshot_params = SnapServiceParams { let snapshot_params = SnapServiceParams {
engine: spec.engine.clone(), engine: spec.engine.clone(),
genesis_block: spec.genesis_block(), genesis_block: spec.genesis_block(),
restoration_db_handler: restoration_db_handler, restoration_db_handler,
pruning: pruning, pruning,
channel: io_service.channel(), channel: io_service.channel(),
snapshot_root: snapshot_path.into(), snapshot_root: snapshot_path.into(),
client: client.clone(), client: client.clone(),

View File

@ -54,6 +54,7 @@
//! ``` //! ```
extern crate account_db; extern crate account_db;
extern crate account_state;
extern crate ansi_term; extern crate ansi_term;
extern crate common_types as types; extern crate common_types as types;
extern crate crossbeam_utils; extern crate crossbeam_utils;
@ -97,8 +98,8 @@ extern crate parity_util_mem;
extern crate parity_util_mem as malloc_size_of; extern crate parity_util_mem as malloc_size_of;
extern crate rustc_hex; extern crate rustc_hex;
extern crate serde; extern crate serde;
extern crate state_db;
extern crate stats; extern crate stats;
extern crate account_state;
extern crate time_utils; extern crate time_utils;
extern crate trace; extern crate trace;
extern crate triehash_ethereum as triehash; extern crate triehash_ethereum as triehash;
@ -163,7 +164,6 @@ pub mod spec;
pub mod verification; pub mod verification;
mod externalities; mod externalities;
mod state_db;
mod transaction_ext; mod transaction_ext;
mod tx_filter; mod tx_filter;
@ -177,4 +177,3 @@ pub mod test_helpers;
pub use executive::contract_address; pub use executive::contract_address;
pub use evm::CreateContractAddress; pub use evm::CreateContractAddress;
pub use trie::TrieSpec; pub use trie::TrieSpec;
pub use state_db::StateDB;

View File

@ -83,9 +83,9 @@ impl Machine {
pub fn regular(params: CommonParams, builtins: BTreeMap<Address, Builtin>) -> Machine { pub fn regular(params: CommonParams, builtins: BTreeMap<Address, Builtin>) -> Machine {
let tx_filter = TransactionFilter::from_params(&params).map(Arc::new); let tx_filter = TransactionFilter::from_params(&params).map(Arc::new);
Machine { Machine {
params: params, params,
builtins: Arc::new(builtins), builtins: Arc::new(builtins),
tx_filter: tx_filter, tx_filter,
ethash_extensions: None, ethash_extensions: None,
schedule_rules: None, schedule_rules: None,
} }

View File

@ -543,8 +543,8 @@ fn load_from(spec_params: SpecParams, s: ethjson::spec::Spec) -> Result<Spec, Er
gas_used: g.gas_used, gas_used: g.gas_used,
timestamp: g.timestamp, timestamp: g.timestamp,
extra_data: g.extra_data, extra_data: g.extra_data,
seal_rlp: seal_rlp, seal_rlp,
hardcoded_sync: hardcoded_sync, hardcoded_sync,
constructors: s.accounts constructors: s.accounts
.constructors() .constructors()
.into_iter() .into_iter()

View File

@ -0,0 +1,28 @@
[package]
description = "State database"
name = "state-db"
version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
license = "GPL-3.0"
edition = "2018"
[dependencies]
account-state = { path = "../account-state" }
bloom_journal = { package = "ethcore-bloom-journal", path = "../../util/bloom" }
common-types = { path = "../types"}
ethcore-db = { path = "../db" }
ethereum-types = "0.6.0"
hash-db = "0.12.4"
keccak-hash = "0.2.0"
keccak-hasher = { path = "../../util/keccak-hasher" }
journaldb = { path = "../../util/journaldb" }
kvdb = "0.1.0"
log = "0.4.6"
lru-cache = "0.1.2"
memory-cache = { path = "../../util/memory-cache" }
parking_lot = "0.8.0"
[dev-dependencies]
env_logger = "0.5"
# Used for test helpers
ethcore = { path = "..", features = ["test-helpers"] }

View File

@ -16,24 +16,25 @@
//! State database abstraction. For more info, see the doc for `StateDB` //! State database abstraction. For more info, see the doc for `StateDB`
use std::collections::{VecDeque, HashSet}; use std::collections::{HashSet, VecDeque};
use std::io; use std::io;
use std::sync::Arc; use std::sync::Arc;
use bloom_journal::{Bloom, BloomJournal}; use ethereum_types::{Address, H256};
use db::COL_ACCOUNT_BLOOM;
use ethereum_types::{H256, Address};
use hash::keccak;
use hash_db::HashDB; use hash_db::HashDB;
use journaldb::JournalDB; use keccak_hash::keccak;
use keccak_hasher::KeccakHasher; use kvdb::{DBTransaction, DBValue, KeyValueDB};
use kvdb::{KeyValueDB, DBTransaction, DBValue}; use log::trace;
use lru_cache::LruCache; use lru_cache::LruCache;
use memory_cache::MemoryLruCache;
use parking_lot::Mutex; use parking_lot::Mutex;
use types::BlockNumber;
use account_state::{self, Account}; use account_state::{self, Account};
use bloom_journal::{Bloom, BloomJournal};
use common_types::BlockNumber;
use ethcore_db::COL_ACCOUNT_BLOOM;
use journaldb::JournalDB;
use keccak_hasher::KeccakHasher;
use memory_cache::MemoryLruCache;
/// Value used to initialize bloom bitmap size. /// Value used to initialize bloom bitmap size.
/// ///
@ -68,7 +69,7 @@ struct AccountCache {
struct CacheQueueItem { struct CacheQueueItem {
/// Account address. /// Account address.
address: Address, address: Address,
/// Acccount data or `None` if account does not exist. /// Account data or `None` if account does not exist.
account: SyncAccount, account: SyncAccount,
/// Indicates that the account was modified before being /// Indicates that the account was modified before being
/// added to the cache. /// added to the cache.
@ -143,7 +144,7 @@ impl StateDB {
let cache_items = acc_cache_size / ::std::mem::size_of::<Option<Account>>(); let cache_items = acc_cache_size / ::std::mem::size_of::<Option<Account>>();
StateDB { StateDB {
db: db, db,
account_cache: Arc::new(Mutex::new(AccountCache { account_cache: Arc::new(Mutex::new(AccountCache {
accounts: LruCache::new(cache_items), accounts: LruCache::new(cache_items),
modifications: VecDeque::new(), modifications: VecDeque::new(),
@ -151,7 +152,7 @@ impl StateDB {
code_cache: Arc::new(Mutex::new(MemoryLruCache::new(code_cache_size))), code_cache: Arc::new(Mutex::new(MemoryLruCache::new(code_cache_size))),
local_cache: Vec::new(), local_cache: Vec::new(),
account_bloom: Arc::new(Mutex::new(bloom)), account_bloom: Arc::new(Mutex::new(bloom)),
cache_size: cache_size, cache_size,
parent_hash: None, parent_hash: None,
commit_hash: None, commit_hash: None,
commit_number: None, commit_number: None,
@ -159,7 +160,7 @@ impl StateDB {
} }
/// Loads accounts bloom from the database /// Loads accounts bloom from the database
/// This bloom is used to handle request for the non-existant account fast /// This bloom is used to handle request for the non-existent account fast
pub fn load_bloom(db: &dyn KeyValueDB) -> Bloom { pub fn load_bloom(db: &dyn KeyValueDB) -> Bloom {
let hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY) let hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY)
.expect("Low-level database error"); .expect("Low-level database error");
@ -177,7 +178,7 @@ impl StateDB {
let key: [u8; 8] = (i as u64).to_le_bytes(); let key: [u8; 8] = (i as u64).to_le_bytes();
bloom_parts[i] = db.get(COL_ACCOUNT_BLOOM, &key).expect("low-level database error") bloom_parts[i] = db.get(COL_ACCOUNT_BLOOM, &key).expect("low-level database error")
.map(|val| { .map(|val| {
assert!(val.len() == 8, "low-level database error"); assert_eq!(val.len(), 8, "low-level database error");
let mut buff = [0u8; 8]; let mut buff = [0u8; 8];
buff.copy_from_slice(&*val); buff.copy_from_slice(&*val);
u64::from_le_bytes(buff) u64::from_le_bytes(buff)
@ -233,7 +234,7 @@ impl StateDB {
let cache = &mut *cache; let cache = &mut *cache;
// Purge changes from re-enacted and retracted blocks. // Purge changes from re-enacted and retracted blocks.
// Filter out commiting block if any. // Filter out committing block if any.
let mut clear = false; let mut clear = false;
for block in enacted.iter().filter(|h| self.commit_hash.as_ref().map_or(true, |p| *h != p)) { for block in enacted.iter().filter(|h| self.commit_hash.as_ref().map_or(true, |p| *h != p)) {
clear = clear || { clear = clear || {
@ -419,11 +420,11 @@ impl account_state::Backend for StateDB {
self.db.as_hash_db_mut() self.db.as_hash_db_mut()
} }
fn add_to_account_cache(&mut self, addr: Address, data: Option<Account>, modified: bool) { fn add_to_account_cache(&mut self, address: Address, data: Option<Account>, modified: bool) {
self.local_cache.push(CacheQueueItem { self.local_cache.push(CacheQueueItem {
address: addr, address,
account: SyncAccount(data), account: SyncAccount(data),
modified: modified, modified,
}) })
} }
@ -484,10 +485,11 @@ unsafe impl Sync for SyncAccount {}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use ethereum_types::{H256, U256, Address}; use ethereum_types::{Address, H256, U256};
use kvdb::DBTransaction; use kvdb::DBTransaction;
use test_helpers::get_temp_state_db;
use account_state::{Account, Backend}; use account_state::{Account, Backend};
use ethcore::test_helpers::get_temp_state_db;
#[test] #[test]
fn state_db_smoke() { fn state_db_smoke() {