From d4205da48410e204acebfaad6d165f83a45fa3a5 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Thu, 1 Mar 2018 19:53:15 +0100 Subject: [PATCH] Extract the hard dependency on rocksdb from the light client (#8034) * Extract the hard dependency on rocksdb from the light client * Remove TODO --- Cargo.lock | 1 - ethcore/Cargo.toml | 2 +- ethcore/light/Cargo.toml | 3 +-- ethcore/light/src/client/mod.rs | 32 --------------------------- ethcore/light/src/client/service.rs | 25 +++++---------------- ethcore/light/src/lib.rs | 4 ++-- ethcore/src/client/test_client.rs | 14 +++++------- ethcore/src/lib.rs | 4 ++-- parity/blockchain.rs | 22 ++++++++++++++---- parity/run.rs | 22 ++++++++++++++---- sync/src/light_sync/tests/test_net.rs | 8 +++++-- 11 files changed, 59 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b114c3e7..0cc37dba0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,7 +553,6 @@ dependencies = [ "keccak-hash 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", - "kvdb-rocksdb 0.1.0", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", "memorydb 0.1.1", diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 8db9a2748..c64a80187 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -69,9 +69,9 @@ keccak-hash = { path = "../util/hash" } triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } -tempdir = "0.3" [dev-dependencies] +tempdir = "0.3" trie-standardmap = { path = "../util/trie-standardmap" } [features] diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 0a5572fc4..6778a4834 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -35,11 +35,10 @@ stats = { path = "../../util/stats" } keccak-hash = { path = "../../util/hash" } triehash = { path = "../../util/triehash" } kvdb = { path = "../../util/kvdb" } -kvdb-rocksdb = { path = "../../util/kvdb-rocksdb" } -kvdb-memorydb = { path = "../../util/kvdb-memorydb" } memory-cache = { path = "../../util/memory_cache" } [dev-dependencies] +kvdb-memorydb = { path = "../../util/kvdb-memorydb" } tempdir = "0.3" [features] diff --git a/ethcore/light/src/client/mod.rs b/ethcore/light/src/client/mod.rs index 32ddf93fb..4c7ea70ad 100644 --- a/ethcore/light/src/client/mod.rs +++ b/ethcore/light/src/client/mod.rs @@ -36,7 +36,6 @@ use ethereum_types::{H256, U256}; use futures::{IntoFuture, Future}; use kvdb::{self, KeyValueDB}; -use kvdb_rocksdb::CompactionProfile; use self::fetch::ChainDataFetcher; use self::header_chain::{AncestryIter, HeaderChain}; @@ -57,12 +56,6 @@ pub struct Config { pub queue: queue::Config, /// Chain column in database. pub chain_column: Option, - /// Database cache size. `None` => rocksdb default. - pub db_cache_size: Option, - /// State db compaction profile - pub db_compaction: CompactionProfile, - /// Should db have WAL enabled? - pub db_wal: bool, /// Should it do full verification of blocks? pub verify_full: bool, /// Should it check the seal of blocks? @@ -74,9 +67,6 @@ impl Default for Config { Config { queue: Default::default(), chain_column: None, - db_cache_size: None, - db_compaction: CompactionProfile::default(), - db_wal: true, verify_full: true, check_seal: true, } @@ -205,28 +195,6 @@ impl Client { self.listeners.write().push(listener); } - /// Create a new `Client` backed purely in-memory. - /// This will ignore all database options in the configuration. - pub fn in_memory( - config: Config, - spec: &Spec, - fetcher: T, - io_channel: IoChannel, - cache: Arc> - ) -> Self { - let db = ::kvdb_memorydb::create(0); - - Client::new( - config, - Arc::new(db), - None, - spec, - fetcher, - io_channel, - cache - ).expect("New DB creation infallible; qed") - } - /// Import a header to the queue for additional verification. pub fn import_header(&self, header: Header) -> Result { self.queue.import(header).map_err(Into::into) diff --git a/ethcore/light/src/client/service.rs b/ethcore/light/src/client/service.rs index 253eef7f5..8be009a65 100644 --- a/ethcore/light/src/client/service.rs +++ b/ethcore/light/src/client/service.rs @@ -18,15 +18,13 @@ //! Just handles block import messages and passes them to the client. use std::fmt; -use std::path::Path; use std::sync::Arc; use ethcore::db; use ethcore::service::ClientIoMessage; use ethcore::spec::Spec; use io::{IoContext, IoError, IoHandler, IoService}; -use kvdb; -use kvdb_rocksdb::{Database, DatabaseConfig}; +use kvdb::{self, KeyValueDB}; use cache::Cache; use parking_lot::Mutex; @@ -59,19 +57,7 @@ pub struct Service { impl Service { /// Start the service: initialize I/O workers and client itself. - pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, path: &Path, cache: Arc>) -> Result { - - // initialize database. - let mut db_config = DatabaseConfig::with_columns(db::NUM_COLUMNS); - - db_config.memory_budget = config.db_cache_size; - db_config.compaction = config.db_compaction; - db_config.wal = config.db_wal; - - let db = Arc::new(Database::open( - &db_config, - &path.to_str().expect("DB path could not be converted to string.") - ).map_err(Error::Database)?); + pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, db: Arc, cache: Arc>) -> Result { let io_service = IoService::::start().map_err(Error::Io)?; let client = Arc::new(Client::new(config, @@ -123,14 +109,15 @@ mod tests { use client::fetch; use time::Duration; use parking_lot::Mutex; - use tempdir::TempDir; + use kvdb_memorydb; + use ethcore::db::NUM_COLUMNS; #[test] fn it_works() { - let tempdir = TempDir::new("").unwrap(); + let db = Arc::new(kvdb_memorydb::create(NUM_COLUMNS.unwrap_or(0))); let spec = Spec::new_test(); let cache = Arc::new(Mutex::new(Cache::new(Default::default(), Duration::hours(6)))); - Service::start(Default::default(), &spec, fetch::unavailable(), tempdir.path(), cache).unwrap(); + Service::start(Default::default(), &spec, fetch::unavailable(), db, cache).unwrap(); } } diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 0e73969be..31894244f 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -80,9 +80,9 @@ extern crate vm; extern crate keccak_hash as hash; extern crate triehash; extern crate kvdb; -extern crate kvdb_memorydb; -extern crate kvdb_rocksdb; extern crate memory_cache; +#[cfg(test)] +extern crate kvdb_memorydb; #[cfg(test)] extern crate tempdir; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index b80d005b5..cd3ce468e 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -27,11 +27,10 @@ use ethereum_types::{H256, U256, Address}; use parking_lot::RwLock; use journaldb; use kvdb::DBValue; -use kvdb_rocksdb::{Database, DatabaseConfig}; +use kvdb_memorydb; use bytes::Bytes; use rlp::*; use ethkey::{Generator, Random}; -use tempdir::TempDir; use transaction::{self, Transaction, LocalizedTransaction, PendingTransaction, SignedTransaction, Action}; use blockchain::{TreeRoute, BlockReceipts}; use client::{ @@ -352,12 +351,10 @@ impl TestBlockChainClient { } } -pub fn get_temp_state_db() -> (StateDB, TempDir) { - let tempdir = TempDir::new("").unwrap(); - let db = Database::open(&DatabaseConfig::with_columns(NUM_COLUMNS), tempdir.path().to_str().unwrap()).unwrap(); +pub fn get_temp_state_db() -> StateDB { + let db = kvdb_memorydb::create(NUM_COLUMNS.unwrap_or(0)); let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, COL_STATE); - let state_db = StateDB::new(journal_db, 1024 * 1024); - (state_db, tempdir) + StateDB::new(journal_db, 1024 * 1024) } impl MiningBlockChainClient for TestBlockChainClient { @@ -370,8 +367,7 @@ impl MiningBlockChainClient for TestBlockChainClient { fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock { let engine = &*self.spec.engine; let genesis_header = self.spec.genesis_header(); - let (state_db, _tempdir) = get_temp_state_db(); - let db = self.spec.ensure_db_good(state_db, &Default::default()).unwrap(); + let db = self.spec.ensure_db_good(get_temp_state_db(), &Default::default()).unwrap(); let last_hashes = vec![genesis_header.hash()]; let mut open_block = OpenBlock::new( diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 7a4e46adf..9c76b814f 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -115,6 +115,8 @@ extern crate vm; extern crate wasm; extern crate memory_cache; extern crate journaldb; +#[cfg(test)] +extern crate tempdir; #[macro_use] extern crate macros; @@ -130,8 +132,6 @@ extern crate evm; #[cfg(feature = "jit" )] extern crate evmjit; -extern crate tempdir; - pub extern crate ethstore; pub mod account_provider; diff --git a/parity/blockchain.rs b/parity/blockchain.rs index 6f6507b01..602a45730 100644 --- a/parity/blockchain.rs +++ b/parity/blockchain.rs @@ -27,11 +27,13 @@ use bytes::ToPretty; use rlp::PayloadInfo; use ethcore::service::ClientService; use ethcore::client::{Mode, DatabaseCompactionProfile, VMType, BlockImportError, BlockChainClient, BlockId}; +use ethcore::db::NUM_COLUMNS; use ethcore::error::ImportError; use ethcore::miner::Miner; use ethcore::verification::queue::VerifierSettings; use cache::CacheConfig; use informant::{Informant, FullNodeInformantData, MillisecondDuration}; +use kvdb_rocksdb::{Database, DatabaseConfig}; use params::{SpecType, Pruning, Switch, tracing_switch_to_bool, fatdb_switch_to_bool}; use helpers::{to_client_config, execute_upgrades}; use dir::Directories; @@ -197,9 +199,6 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> { let mut config = LightClientConfig { queue: Default::default(), chain_column: ::ethcore::db::COL_LIGHT_CHAIN, - db_cache_size: Some(cmd.cache_config.blockchain() as usize * 1024 * 1024), - db_compaction: compaction, - db_wal: cmd.wal, verify_full: true, check_seal: cmd.check_seal, }; @@ -207,9 +206,24 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> { config.queue.max_mem_use = cmd.cache_config.queue() as usize * 1024 * 1024; config.queue.verifier_settings = cmd.verifier_settings; + // initialize database. + let db = { + let db_config = DatabaseConfig { + memory_budget: Some(cmd.cache_config.blockchain() as usize * 1024 * 1024), + compaction: compaction, + wal: cmd.wal, + .. DatabaseConfig::with_columns(NUM_COLUMNS) + }; + + Arc::new(Database::open( + &db_config, + &client_path.to_str().expect("DB path could not be converted to string.") + ).map_err(|e| format!("Failed to open database: {}", e))?) + }; + // TODO: could epoch signals be avilable at the end of the file? let fetch = ::light::client::fetch::unavailable(); - let service = LightClientService::start(config, &spec, fetch, &client_path, cache) + let service = LightClientService::start(config, &spec, fetch, db, cache) .map_err(|e| format!("Failed to start client: {}", e))?; // free up the spec in memory. diff --git a/parity/run.rs b/parity/run.rs index d8abf40c0..a1d067052 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -24,6 +24,7 @@ use ansi_term::Colour; use ctrlc::CtrlC; use ethcore::account_provider::{AccountProvider, AccountProviderSettings}; use ethcore::client::{Client, Mode, DatabaseCompactionProfile, VMType, BlockChainClient}; +use ethcore::db::NUM_COLUMNS; use ethcore::ethstore::ethkey; use ethcore::miner::{Miner, MinerService, MinerOptions}; use ethcore::miner::{StratumOptions, Stratum}; @@ -38,6 +39,7 @@ use hash_fetch::fetch::{Fetch, Client as FetchClient}; use hash_fetch; use informant::{Informant, LightNodeInformantData, FullNodeInformantData}; use journaldb::Algorithm; +use kvdb_rocksdb::{Database, DatabaseConfig}; use light::Cache as LightDataCache; use miner::external::ExternalMiner; use node_filter::NodeFilter; @@ -222,9 +224,6 @@ fn execute_light_impl(cmd: RunCmd, can_restart: bool, logger: Arc { // skip full verification because the blocks are bad. config.verify_full = false; let cache = Arc::new(Mutex::new(Cache::new(Default::default(), Duration::hours(6)))); - let client = LightClient::in_memory( + let db = kvdb_memorydb::create(0); + let client = LightClient::new( config, + Arc::new(db), + None, &Spec::new_test(), fetch::unavailable(), // TODO: allow fetch from full nodes. IoChannel::disconnected(), cache - ); + ).expect("New DB creation infallible; qed"); peers.push(Arc::new(Peer::new_light(Arc::new(client)))) }