From 781f763f1f50463117959f9e2d1bf0be83f1a081 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Feb 2016 14:09:39 +0100 Subject: [PATCH] Memory management --- Cargo.lock | 10 ++++++ Cargo.toml | 1 + ethcore/src/block_queue.rs | 65 ++++++++++++++++++++++++++++++----- ethcore/src/blockchain.rs | 66 +++++++++++++++++------------------- ethcore/src/client.rs | 28 ++++++++++----- ethcore/src/lib.rs | 6 ++-- ethcore/src/service.rs | 9 ++--- ethcore/src/tests/client.rs | 12 +++---- ethcore/src/tests/helpers.rs | 14 ++++---- parity/main.rs | 35 +++++++++++++------ sync/src/tests/helpers.rs | 4 ++- 11 files changed, 167 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1ef76816..695b060f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,7 @@ dependencies = [ "ethsync 0.9.99", "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -255,6 +256,7 @@ dependencies = [ "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", "ethcore-util 0.9.99", + "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -526,6 +528,14 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "odds" version = "0.2.12" diff --git a/Cargo.toml b/Cargo.toml index 2de097ad9..9b8ec6405 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } daemonize = "0.2" ethcore-devtools = { path = "devtools" } +number_prefix = "0.2" [features] default = ["rpc"] diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index c39f158f0..0f5985f06 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -28,6 +28,28 @@ use service::*; use client::BlockStatus; use util::panics::*; +known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); + +/// Block queue configuration +#[derive(Debug)] +pub struct BlockQueueConfig { + /// Maximum number of blocks to keep in unverified queue. + /// When the limit is reached, is_full returns true. + pub max_queue_size: usize, + /// Maximum heap memory to use. + /// When the limit is reached, is_full returns true. + pub max_mem_use: usize, +} + +impl Default for BlockQueueConfig { + fn default() -> Self { + BlockQueueConfig { + max_queue_size: 30000, + max_mem_use: 50 * 1024 * 1024, + } + } +} + /// Block queue status #[derive(Debug)] pub struct BlockQueueInfo { @@ -37,6 +59,12 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, /// Number of blocks being verified pub verifying_queue_size: usize, + /// Configured maximum number of blocks in the queue + pub max_queue_size: usize, + /// Configured maximum number of bytes to use + pub max_mem_use: usize, + /// Heap memory used in bytes + pub mem_used: usize, } impl BlockQueueInfo { @@ -48,7 +76,8 @@ impl BlockQueueInfo { /// Indicates that queue is full pub fn is_full(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > MAX_UNVERIFIED_QUEUE_SIZE + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || + self.mem_used > self.max_mem_use } /// Indicates that queue is empty @@ -68,7 +97,9 @@ pub struct BlockQueue { deleting: Arc, ready_signal: Arc, empty: Arc, - processing: RwLock> + processing: RwLock>, + max_queue_size: usize, + max_mem_use: usize, } struct UnVerifiedBlock { @@ -106,11 +137,9 @@ struct Verification { bad: HashSet, } -const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; - impl BlockQueue { /// Creates a new queue instance. - pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { + pub fn new(config: BlockQueueConfig, engine: Arc>, message_channel: IoChannel) -> BlockQueue { let verification = Arc::new(Mutex::new(Verification::default())); let more_to_verify = Arc::new(Condvar::new()); let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); @@ -133,7 +162,7 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) + BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) .expect("Error starting block verification thread") @@ -149,6 +178,8 @@ impl BlockQueue { deleting: deleting.clone(), processing: RwLock::new(HashSet::new()), empty: empty.clone(), + max_queue_size: config.max_queue_size, + max_mem_use: config.max_mem_use, } } @@ -334,8 +365,26 @@ impl BlockQueue { verified_queue_size: verification.verified.len(), unverified_queue_size: verification.unverified.len(), verifying_queue_size: verification.verifying.len(), + max_queue_size: self.max_queue_size, + max_mem_use: self.max_mem_use, + mem_used: + verification.unverified.heap_size_of_children() + + verification.verifying.heap_size_of_children() + + verification.verified.heap_size_of_children(), + // TODO: https://github.com/servo/heapsize/pull/50 + //+ self.processing.read().unwrap().heap_size_of_children(), } } + + pub fn collect_garbage(&self) { + { + let mut verification = self.verification.lock().unwrap(); + verification.unverified.shrink_to_fit(); + verification.verifying.shrink_to_fit(); + verification.verified.shrink_to_fit(); + } + self.processing.write().unwrap().shrink_to_fit(); + } } impl MayPanic for BlockQueue { @@ -367,7 +416,7 @@ mod tests { fn get_test_queue() -> BlockQueue { let spec = get_test_spec(); let engine = spec.to_engine().unwrap(); - BlockQueue::new(Arc::new(engine), IoChannel::disconnected()) + BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()) } #[test] @@ -375,7 +424,7 @@ mod tests { // TODO better test let spec = Spec::new_test(); let engine = spec.to_engine().unwrap(); - let _ = BlockQueue::new(Arc::new(engine), IoChannel::disconnected()); + let _ = BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()); } #[test] diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index cc9ff56fd..6907369f4 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -23,6 +23,24 @@ use extras::*; use transaction::*; use views::*; +/// Blockchain configuration. +#[derive(Debug)] +pub struct BlockChainConfig { + /// Preferred cache size in bytes. + pub pref_cache_size: usize, + /// Maximum cache size in bytes. + pub max_cache_size: usize, +} + +impl Default for BlockChainConfig { + fn default() -> Self { + BlockChainConfig { + pref_cache_size: 1 << 14, + max_cache_size: 1 << 20, + } + } +} + /// Represents a tree route between `from` block and `to` block: pub struct TreeRoute { /// A vector of hashes of all blocks, ordered from `from` to `to`. @@ -50,7 +68,7 @@ pub struct CacheSize { impl CacheSize { /// Total amount used by the cache. - fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } + pub fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } } /// Information about best block gathered together @@ -220,33 +238,7 @@ const COLLECTION_QUEUE_SIZE: usize = 8; impl BlockChain { /// Create new instance of blockchain from given Genesis - /// - /// ```rust - /// extern crate ethcore_util as util; - /// extern crate ethcore; - /// use std::env; - /// use std::str::FromStr; - /// use ethcore::spec::*; - /// use ethcore::blockchain::*; - /// use ethcore::ethereum; - /// use util::hash::*; - /// use util::uint::*; - /// - /// fn main() { - /// let spec = ethereum::new_frontier(); - /// - /// let mut dir = env::temp_dir(); - /// dir.push(H32::random().hex()); - /// - /// let bc = BlockChain::new(&spec.genesis_block(), &dir); - /// - /// let genesis_hash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"; - /// assert_eq!(bc.genesis_hash(), H256::from_str(genesis_hash).unwrap()); - /// assert!(bc.is_known(&bc.genesis_hash())); - /// assert_eq!(bc.genesis_hash(), bc.block_hash(0).unwrap()); - /// } - /// ``` - pub fn new(genesis: &[u8], path: &Path) -> BlockChain { + pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { // open extras db let mut extras_path = path.to_path_buf(); extras_path.push("extras"); @@ -261,8 +253,8 @@ impl BlockChain { (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: 1 << 14, - max_cache_size: 1 << 20, + pref_cache_size: config.pref_cache_size, + max_cache_size: config.max_cache_size, best_block: RwLock::new(BestBlock::new()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), @@ -536,6 +528,8 @@ impl BlockChain { } /// Returns true if transaction is known. + // TODO: Use me + #[allow(dead_code)] pub fn is_known_transaction(&self, hash: &H256) -> bool { self.query_extras_exist(hash, &self.transaction_addresses) } @@ -556,6 +550,8 @@ impl BlockChain { } /// Get the transactions' log blooms of a block. + // TODO: Use me + #[allow(dead_code)] pub fn log_blooms(&self, hash: &H256) -> Option { self.query_extras(hash, &self.block_logs) } @@ -671,7 +667,7 @@ mod tests { let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0925002c3260b44e44c3edebad1cc442142b03020209df1ab8bb86752edbd2cd7a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a0363659b251bf8b819179874c8cce7b9b983d7f3704cbb58a3b334431f7032871889032d09c281e1236c0c0".from_hex().unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); let genesis_hash = H256::from_str("3caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942").unwrap(); @@ -715,7 +711,7 @@ mod tests { let best_block_hash = H256::from_str("c208f88c9f5bf7e00840439742c12e5226d9752981f3ec0521bdcb6dd08af277").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1); bc.insert_block(&b2); bc.insert_block(&b3a); @@ -794,14 +790,14 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), genesis_hash); bc.insert_block(&b1); assert_eq!(bc.best_block_hash(), b1_hash); } { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), b1_hash); } } @@ -854,7 +850,7 @@ mod tests { let b1_hash = H256::from_str("f53f268d23a71e85c7d6d83a9504298712b84c1a2ba220441c86eeda0bf0b6e3").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1); let transactions = bc.transactions(&b1_hash).unwrap(); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c3ec4b4d0..333917e4a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -19,7 +19,7 @@ use util::*; use util::panics::*; use rocksdb::{Options, DB, DBCompactionStyle}; -use blockchain::{BlockChain, BlockProvider, CacheSize}; +use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; use header::BlockNumber; @@ -27,14 +27,16 @@ use state::State; use spec::Spec; use engine::Engine; use views::HeaderView; -use block_queue::{BlockQueue, BlockQueueInfo}; +use block_queue::BlockQueue; use service::{NetSyncMessage, SyncMessage}; use env_info::LastHashes; use verification::*; use block::*; use transaction::LocalizedTransaction; use extras::TransactionAddress; -pub use blockchain::TreeRoute; +pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; +pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; + /// Uniquely identifies block. #[derive(Debug, PartialEq, Clone)] @@ -73,7 +75,16 @@ pub enum BlockStatus { Unknown, } -/// Information about the blockchain gthered together. +/// Client configuration. Includes configs for all sub-systems. +#[derive(Debug, Default)] +pub struct ClientConfig { + /// Block queue configuration. + pub queue: BlockQueueConfig, + /// Blockchain configuration. + pub blockchain: BlockChainConfig, +} + +/// Information about the blockchain gathered together. #[derive(Debug)] pub struct BlockChainInfo { /// Blockchain difficulty. @@ -183,14 +194,14 @@ const CLIENT_DB_VER_STR: &'static str = "2.1"; impl Client { /// Create a new client with given spec and DB path. - pub fn new(spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); let path = dir.as_path(); let gb = spec.genesis_block(); - let chain = Arc::new(RwLock::new(BlockChain::new(&gb, path))); + let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); let mut opts = Options::new(); opts.set_max_open_files(256); opts.create_if_missing(true); @@ -223,7 +234,7 @@ impl Client { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } - let block_queue = BlockQueue::new(engine.clone(), message_channel); + let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel); let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); @@ -330,7 +341,7 @@ impl Client { } /// Get info on the cache. - pub fn cache_info(&self) -> CacheSize { + pub fn blockchain_cache_info(&self) -> BlockChainCacheSize { self.chain.read().unwrap().cache_size() } @@ -342,6 +353,7 @@ impl Client { /// Tick the client. pub fn tick(&self) { self.chain.read().unwrap().collect_garbage(); + self.block_queue.read().unwrap().collect_garbage(); } /// Set up the cache behaviour. diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 3d43fd725..17bd52159 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -85,7 +85,7 @@ #[macro_use] extern crate lazy_static; extern crate rustc_serialize; extern crate rocksdb; -extern crate heapsize; +#[macro_use] extern crate heapsize; extern crate crypto; extern crate time; extern crate env_logger; @@ -96,8 +96,6 @@ extern crate crossbeam; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; -pub mod blockchain; -pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -129,6 +127,8 @@ mod substate; mod executive; mod externalities; mod verification; +mod block_queue; +mod blockchain; #[cfg(test)] mod tests; diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 534aab49d..ea80e7c80 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -20,7 +20,7 @@ use util::*; use util::panics::*; use spec::Spec; use error::*; -use client::Client; +use client::{Client, ClientConfig}; /// Message type for external and internal events #[derive(Clone)] @@ -43,14 +43,14 @@ pub struct ClientService { impl ClientService { /// Start the service in a separate thread. - pub fn start(spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { + pub fn start(config: ClientConfig, spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { let panic_handler = PanicHandler::new_in_arc(); let mut net_service = try!(NetworkService::start(net_config)); panic_handler.forward_from(&net_service); info!("Starting {}", net_service.host_info()); info!("Configured for {} using {} engine", spec.name, spec.engine_name); - let client = try!(Client::new(spec, db_path, net_service.io().channel())); + let client = try!(Client::new(config, spec, db_path, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { client: client.clone() @@ -130,12 +130,13 @@ mod tests { use tests::helpers::*; use util::network::*; use devtools::*; + use client::ClientConfig; #[test] fn it_can_be_started() { let spec = get_test_spec(); let temp_path = RandomTempPath::new(); - let service = ClientService::start(spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); + let service = ClientService::start(ClientConfig::default(), spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); assert!(service.is_ok()); } } diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index af25d1b72..83df81fa2 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client, BlockId}; +use client::{BlockChainClient, Client, ClientConfig, BlockId}; use tests::helpers::*; use common::*; use devtools::*; @@ -22,14 +22,14 @@ use devtools::*; #[test] fn created() { let dir = RandomTempPath::new(); - let client_result = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()); + let client_result = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()); assert!(client_result.is_ok()); } #[test] fn imports_from_empty() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); client.import_verified_blocks(&IoChannel::disconnected()); client.flush_queue(); } @@ -37,7 +37,7 @@ fn imports_from_empty() { #[test] fn imports_good_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let good_block = get_good_dummy_block(); if let Err(_) = client.import_block(good_block) { panic!("error importing block being good by definition"); @@ -52,7 +52,7 @@ fn imports_good_block() { #[test] fn query_none_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let non_existant = client.block_header(BlockId::Number(188)); assert!(non_existant.is_none()); @@ -104,5 +104,5 @@ fn can_collect_garbage() { let client_result = generate_dummy_client(100); let client = client_result.reference(); client.tick(); - assert!(client.cache_info().blocks < 100 * 1024); + assert!(client.blockchain_cache_info().blocks < 100 * 1024); } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 56653e820..510833f4d 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client}; +use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; -use blockchain::{BlockChain}; +use blockchain::{BlockChain, BlockChainConfig}; use state::*; use rocksdb::*; use evm::{Schedule, Factory}; @@ -135,7 +135,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[&SignedTrans pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let test_spec = get_test_spec(); let test_engine = test_spec.to_engine().unwrap(); let state_root = test_engine.spec().genesis_header().state_root; @@ -173,7 +173,7 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); for block in &blocks { if let Err(_) = client.import_block(block.clone()) { panic!("panic importing block which is well-formed"); @@ -190,7 +190,7 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block(block_order, bc.best_block_hash())); } @@ -203,7 +203,7 @@ pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None)); } @@ -216,7 +216,7 @@ pub fn generate_dummy_blockchain_with_extra(block_number: u32) -> GuardedTempRes pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); GuardedTempResult:: { _temp: temp, diff --git a/parity/main.rs b/parity/main.rs index e95f38f13..fad2be840 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -31,6 +31,7 @@ extern crate ctrlc; extern crate fdlimit; extern crate daemonize; extern crate time; +extern crate number_prefix; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; @@ -47,10 +48,10 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethcore::blockchain::CacheSize; use ethsync::{EthSync, SyncConfig}; use docopt::Docopt; use daemonize::Daemonize; +use number_prefix::{binary_prefix, Standalone, Prefixed}; const USAGE: &'static str = " Parity. Ethereum Client. @@ -78,6 +79,7 @@ Options: --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in bytes [default: 262144]. + --queue-max-size BYTES Specify the maximum size of memory to use for block queue [default: 52428800]. -j --jsonrpc Enable the JSON-RPC API sever. --jsonrpc-url URL Specify URL for JSON-RPC API server [default: 127.0.0.1:8545]. @@ -105,6 +107,7 @@ struct Args { flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, + flag_queue_max_size: usize, flag_jsonrpc: bool, flag_jsonrpc_url: String, flag_logging: Option, @@ -285,9 +288,12 @@ impl Configuration { sync_config.network_id = spec.network_id(); // Build client - let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); + let mut client_config = ClientConfig::default(); + client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; + client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.queue.max_mem_use = self.args.flag_queue_max_size; + let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); - client.configure_cache(self.args.flag_cache_pref_size, self.args.flag_cache_max_size); // Sync let sync = EthSync::register(service.network(), sync_config, client); @@ -331,7 +337,7 @@ fn main() { struct Informant { chain_info: RwLock>, - cache_info: RwLock>, + cache_info: RwLock>, report: RwLock>, } @@ -346,18 +352,26 @@ impl Default for Informant { } impl Informant { + + fn format_bytes(b: usize) -> String { + match binary_prefix(b as f64) { + Standalone(bytes) => format!("{} bytes", bytes), + Prefixed(prefix, n) => format!("{:.0} {}B", n, prefix), + } + } + pub fn tick(&self, client: &Client, sync: &EthSync) { // 5 seconds betwen calls. TODO: calculate this properly. let dur = 5usize; let chain_info = client.chain_info(); let queue_info = client.queue_info(); - let cache_info = client.cache_info(); + let cache_info = client.blockchain_cache_info(); let report = client.report(); let sync_info = sync.status(); - if let (_, &Some(ref last_cache_info), &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { - println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// {} ({}) bl {} ({}) ex ]", + if let (_, _, &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { + println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// mem: {} chain, {} queue, {} sync ]", chain_info.best_block_number, chain_info.best_block_hash, (report.blocks_imported - last_report.blocks_imported) / dur, @@ -370,10 +384,9 @@ impl Informant { queue_info.unverified_queue_size, queue_info.verified_queue_size, - cache_info.blocks, - cache_info.blocks as isize - last_cache_info.blocks as isize, - cache_info.block_details, - cache_info.block_details as isize - last_cache_info.block_details as isize + Informant::format_bytes(cache_info.total()), + Informant::format_bytes(queue_info.mem_used), + Informant::format_bytes(sync_info.mem_used), ); } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 6e92184c8..05b190573 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -16,7 +16,7 @@ use util::*; use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId}; -use ethcore::block_queue::BlockQueueInfo; +use ethcore::BlockQueueInfo; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; @@ -242,6 +242,8 @@ impl BlockChainClient for TestBlockChainClient { verified_queue_size: 0, unverified_queue_size: 0, verifying_queue_size: 0, + max_unverified: 0, + mem_used: 0, } }