From cb3608c6d3558ad2d39c2151d26fefeb25e8ae90 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 24 Feb 2016 21:23:58 +0100 Subject: [PATCH 1/6] Limit download ahead --- ethcore/src/spec.rs | 6 ++++ parity/main.rs | 6 ++-- sync/src/chain.rs | 63 +++++++++++++++++++++++------------- sync/src/lib.rs | 26 ++++++++++++--- sync/src/range_collection.rs | 2 +- sync/src/tests/helpers.rs | 5 +-- 6 files changed, 76 insertions(+), 32 deletions(-) diff --git a/ethcore/src/spec.rs b/ethcore/src/spec.rs index 5714ca734..38a0dda53 100644 --- a/ethcore/src/spec.rs +++ b/ethcore/src/spec.rs @@ -58,6 +58,8 @@ pub struct Spec { /// Known nodes on the network in enode format. pub nodes: Vec, + /// Network ID + pub network_id: U256, /// Parameters concerning operation of the specific engine we're using. /// Maps the parameter name to an RLP-encoded value. @@ -120,6 +122,9 @@ impl Spec { /// Get the known knodes of the network in enode format. pub fn nodes(&self) -> &Vec { &self.nodes } + /// Get the configured Network ID. + pub fn network_id(&self) -> U256 { self.network_id } + /// Get the header of the genesis block. pub fn genesis_header(&self) -> Header { Header { @@ -250,6 +255,7 @@ impl FromJson for Spec { engine_name: json["engineName"].as_string().unwrap().to_owned(), engine_params: json_to_rlp_map(&json["params"]), nodes: nodes, + network_id: U256::from_str(&json["params"]["networkID"].as_string().unwrap()[2..]).unwrap(), builtins: builtins, parent_hash: H256::from_str(&genesis["parentHash"].as_string().unwrap()[2..]).unwrap(), author: Address::from_str(&genesis["author"].as_string().unwrap()[2..]).unwrap(), diff --git a/parity/main.rs b/parity/main.rs index 1b6a59a93..e95f38f13 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -48,7 +48,7 @@ use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; use ethcore::blockchain::CacheSize; -use ethsync::EthSync; +use ethsync::{EthSync, SyncConfig}; use docopt::Docopt; use daemonize::Daemonize; @@ -281,6 +281,8 @@ impl Configuration { let spec = self.spec(); let net_settings = self.net_settings(&spec); + let mut sync_config = SyncConfig::default(); + sync_config.network_id = spec.network_id(); // Build client let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); @@ -288,7 +290,7 @@ impl Configuration { client.configure_cache(self.args.flag_cache_pref_size, self.args.flag_cache_max_size); // Sync - let sync = EthSync::register(service.network(), client); + let sync = EthSync::register(service.network(), sync_config, client); // Setup rpc if self.args.flag_jsonrpc { diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 5c79e08b6..bea17c177 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -40,6 +40,7 @@ use ethcore::block::Block; use io::SyncIo; use time; use std::option::Option; +use super::SyncConfig; impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { @@ -80,9 +81,7 @@ const NODE_DATA_PACKET: u8 = 0x0e; const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; -const NETWORK_ID: U256 = ONE_U256; //TODO: get this from parent - -const CONNECTION_TIMEOUT_SEC: f64 = 10f64; +const CONNECTION_TIMEOUT_SEC: f64 = 5f64; struct Header { /// Header data @@ -203,13 +202,17 @@ pub struct ChainSync { have_common_block: bool, /// Last propagated block number last_send_block_number: BlockNumber, + /// Max blocks to download ahead + max_download_ahead_blocks: usize, + /// Network ID + network_id: U256, } type RlpResponseResult = Result, PacketDecodeError>; impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new() -> ChainSync { + pub fn new(config: SyncConfig) -> ChainSync { ChainSync { state: SyncState::NotSynced, starting_block: 0, @@ -226,6 +229,8 @@ impl ChainSync { syncing_difficulty: U256::from(0u64), have_common_block: false, last_send_block_number: 0, + max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), + network_id: config.network_id, } } @@ -275,7 +280,6 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; - io.chain().clear_queue(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -307,7 +311,7 @@ impl ChainSync { trace!(target: "sync", "Peer {} genesis hash not matched", peer_id); return Ok(()); } - if peer.network_id != NETWORK_ID { + if peer.network_id != self.network_id { io.disable_peer(peer_id); trace!(target: "sync", "Peer {} network id not matched", peer_id); return Ok(()); @@ -436,7 +440,7 @@ impl ChainSync { trace!(target: "sync", "Got body {}", n); } None => { - debug!(target: "sync", "Ignored unknown block body"); + trace!(target: "sync", "Ignored unknown/stale block body"); } } } @@ -608,7 +612,7 @@ impl ChainSync { self.request_headers_by_hash(io, peer_id, &peer_latest, 1, 0, false); } else if self.state == SyncState::Blocks && io.chain().block_status(BlockId::Hash(peer_latest)) == BlockStatus::Unknown { - self.request_blocks(io, peer_id); + self.request_blocks(io, peer_id, false); } } @@ -617,7 +621,7 @@ impl ChainSync { } /// Find some headers or blocks to download for a peer. - fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId) { + fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, ignore_others: bool) { self.clear_peer_download(peer_id); if io.chain().queue_info().is_full() { @@ -637,28 +641,34 @@ impl ChainSync { let mut index: BlockNumber = 0; while index != items.len() as BlockNumber && needed_bodies.len() < MAX_BODIES_TO_REQUEST { let block = start + index; - if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { + if ignore_others || (!self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block)) { needed_bodies.push(items[index as usize].hash.clone()); needed_numbers.push(block); - self.downloading_bodies.insert(block); } index += 1; } } } if !needed_bodies.is_empty() { + let (head, _) = self.headers.range_iter().next().unwrap(); + if needed_numbers.first().unwrap() - head > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading block bodies", peer_id, needed_numbers.first().unwrap(), head); + self.request_blocks(io, peer_id, true); + return; + } + self.downloading_bodies.extend(needed_numbers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, needed_numbers); self.request_bodies(io, peer_id, needed_bodies); } else { // check if need to download headers - let mut start = 0usize; + let mut start = 0; if !self.have_common_block { // download backwards until common block is found 1 header at a time let chain_info = io.chain().chain_info(); - start = chain_info.best_block_number as usize; + start = chain_info.best_block_number; if !self.headers.is_empty() { - start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); + start = min(start, self.headers.range_iter().next().unwrap().0 - 1); } if start == 0 { self.have_common_block = true; //reached genesis @@ -669,6 +679,7 @@ impl ChainSync { if self.have_common_block { let mut headers: Vec = Vec::new(); let mut prev = self.current_base_block() + 1; + let head = self.headers.range_iter().next().map(|(h, _)| h); for (next, ref items) in self.headers.range_iter() { if !headers.is_empty() { break; @@ -679,9 +690,8 @@ impl ChainSync { } let mut block = prev; while block < next && headers.len() < MAX_HEADERS_TO_REQUEST { - if !self.downloading_headers.contains(&(block as BlockNumber)) { + if ignore_others || !self.downloading_headers.contains(&(block as BlockNumber)) { headers.push(block as BlockNumber); - self.downloading_headers.insert(block as BlockNumber); } block += 1; } @@ -689,17 +699,23 @@ impl ChainSync { } if !headers.is_empty() { - start = headers[0] as usize; + start = headers[0]; + if head.is_some() && start > head.unwrap() && start - head.unwrap() > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading headers", peer_id, start, head.unwrap()); + self.request_blocks(io, peer_id, true); + return; + } let count = headers.len(); + self.downloading_headers.extend(headers.iter()); replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, headers); - assert!(!self.headers.have_item(&(start as BlockNumber))); - self.request_headers_by_number(io, peer_id, start as BlockNumber, count, 0, false); + assert!(!self.headers.have_item(&start)); + self.request_headers_by_number(io, peer_id, start, count, 0, false); } } else { // continue search for common block - self.downloading_headers.insert(start as BlockNumber); - self.request_headers_by_number(io, peer_id, start as BlockNumber, 1, 0, false); + self.downloading_headers.insert(start); + self.request_headers_by_number(io, peer_id, start, 1, 0, false); } } } @@ -891,7 +907,7 @@ impl ChainSync { let mut packet = RlpStream::new_list(5); let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); - packet.append(&NETWORK_ID); //TODO: network id + packet.append(&self.network_id); packet.append(&chain.total_difficulty); packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); @@ -1221,6 +1237,7 @@ impl ChainSync { mod tests { use tests::helpers::*; use super::*; + use ::SyncConfig; use util::*; use super::{PeerInfo, PeerAsking}; use ethcore::header::*; @@ -1334,7 +1351,7 @@ mod tests { } fn dummy_sync_with_peer(peer_latest_hash: H256) -> ChainSync { - let mut sync = ChainSync::new(); + let mut sync = ChainSync::new(SyncConfig::default()); sync.peers.insert(0, PeerInfo { protocol_version: 0, diff --git a/sync/src/lib.rs b/sync/src/lib.rs index fd586409a..397a09f47 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -35,14 +35,14 @@ //! use std::sync::Arc; //! use util::network::{NetworkService, NetworkConfiguration}; //! use ethcore::client::Client; -//! use ethsync::EthSync; +//! use ethsync::{EthSync, SyncConfig}; //! use ethcore::ethereum; //! //! fn main() { //! let mut service = NetworkService::start(NetworkConfiguration::new()).unwrap(); //! let dir = env::temp_dir(); //! let client = Client::new(ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); -//! EthSync::register(&mut service, client); +//! EthSync::register(&mut service, SyncConfig::default(), client); //! } //! ``` @@ -60,6 +60,7 @@ use std::sync::*; use ethcore::client::Client; use util::network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId}; use util::TimerToken; +use util::{U256, ONE_U256}; use chain::ChainSync; use ethcore::service::SyncMessage; use io::NetSyncIo; @@ -71,6 +72,23 @@ mod range_collection; #[cfg(test)] mod tests; +/// Sync configuration +pub struct SyncConfig { + /// Max blocks to download ahead + pub max_download_ahead_blocks: usize, + /// Network ID + pub network_id: U256, +} + +impl Default for SyncConfig { + fn default() -> SyncConfig { + SyncConfig { + max_download_ahead_blocks: 20000, + network_id: ONE_U256, + } + } +} + /// Ethereum network protocol handler pub struct EthSync { /// Shared blockchain client. TODO: this should evetually become an IPC endpoint @@ -83,10 +101,10 @@ pub use self::chain::{SyncStatus, SyncState}; impl EthSync { /// Creates and register protocol with the network service - pub fn register(service: &mut NetworkService, chain: Arc) -> Arc { + pub fn register(service: &mut NetworkService, config: SyncConfig, chain: Arc) -> Arc { let sync = Arc::new(EthSync { chain: chain, - sync: RwLock::new(ChainSync::new()), + sync: RwLock::new(ChainSync::new(config)), }); service.register_protocol(sync.clone(), "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); sync diff --git a/sync/src/range_collection.rs b/sync/src/range_collection.rs index c3333ab63..0a1bb6c6f 100644 --- a/sync/src/range_collection.rs +++ b/sync/src/range_collection.rs @@ -207,7 +207,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + } #[test] -#[allow(cyclomatic_complexity)] +#[cfg_attr(dev, allow(cyclomatic_complexity))] fn test_range() { use std::cmp::{Ordering}; diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 8c8b3b10a..6e92184c8 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -20,7 +20,8 @@ use ethcore::block_queue::BlockQueueInfo; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; -use chain::{ChainSync}; +use chain::ChainSync; +use ::SyncConfig; use ethcore::receipt::Receipt; use ethcore::transaction::LocalizedTransaction; @@ -330,7 +331,7 @@ impl TestNet { for _ in 0..n { net.peers.push(TestPeer { chain: TestBlockChainClient::new(), - sync: ChainSync::new(), + sync: ChainSync::new(SyncConfig::default()), queue: VecDeque::new(), }); } From 1a73d703342ee3b62f807637aa6e0fa58a729f7d Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 24 Feb 2016 22:37:28 +0100 Subject: [PATCH 2/6] Report memory usage --- sync/Cargo.toml | 1 + sync/src/chain.rs | 14 +++++++++++++- sync/src/lib.rs | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 7a81c7d97..26a7d463c 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -15,6 +15,7 @@ log = "0.3" env_logger = "0.3" time = "0.1.34" rand = "0.3.13" +heapsize = "0.3" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index bea17c177..01446adaf 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -39,9 +39,10 @@ use ethcore::error::*; use ethcore::block::Block; use io::SyncIo; use time; -use std::option::Option; use super::SyncConfig; +known_heap_size!(0, PeerInfo, Header, HeaderId); + impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { *self as usize @@ -134,6 +135,8 @@ pub struct SyncStatus { pub num_peers: usize, /// Total number of active peers pub num_active_peers: usize, + /// Heap memory used in bytes + pub mem_used: usize, } #[derive(PartialEq, Eq, Debug, Clone)] @@ -246,6 +249,15 @@ impl ChainSync { blocks_total: match self.highest_block { None => 0, Some(x) => x - self.starting_block }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), + mem_used: + // TODO: https://github.com/servo/heapsize/pull/50 + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + + self.header_ids.heap_size_of_children(), } } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 397a09f47..9bf715f07 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -54,6 +54,8 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; +#[macro_use] +extern crate heapsize; use std::ops::*; use std::sync::*; From 781f763f1f50463117959f9e2d1bf0be83f1a081 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Feb 2016 14:09:39 +0100 Subject: [PATCH 3/6] Memory management --- Cargo.lock | 10 ++++++ Cargo.toml | 1 + ethcore/src/block_queue.rs | 65 ++++++++++++++++++++++++++++++----- ethcore/src/blockchain.rs | 66 +++++++++++++++++------------------- ethcore/src/client.rs | 28 ++++++++++----- ethcore/src/lib.rs | 6 ++-- ethcore/src/service.rs | 9 ++--- ethcore/src/tests/client.rs | 12 +++---- ethcore/src/tests/helpers.rs | 14 ++++---- parity/main.rs | 35 +++++++++++++------ sync/src/tests/helpers.rs | 4 ++- 11 files changed, 167 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1ef76816..695b060f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,7 @@ dependencies = [ "ethsync 0.9.99", "fdlimit 0.1.0", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -255,6 +256,7 @@ dependencies = [ "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 0.9.99", "ethcore-util 0.9.99", + "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", @@ -526,6 +528,14 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "odds" version = "0.2.12" diff --git a/Cargo.toml b/Cargo.toml index 2de097ad9..9b8ec6405 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ ethcore-rpc = { path = "rpc", optional = true } fdlimit = { path = "util/fdlimit" } daemonize = "0.2" ethcore-devtools = { path = "devtools" } +number_prefix = "0.2" [features] default = ["rpc"] diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index c39f158f0..0f5985f06 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -28,6 +28,28 @@ use service::*; use client::BlockStatus; use util::panics::*; +known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); + +/// Block queue configuration +#[derive(Debug)] +pub struct BlockQueueConfig { + /// Maximum number of blocks to keep in unverified queue. + /// When the limit is reached, is_full returns true. + pub max_queue_size: usize, + /// Maximum heap memory to use. + /// When the limit is reached, is_full returns true. + pub max_mem_use: usize, +} + +impl Default for BlockQueueConfig { + fn default() -> Self { + BlockQueueConfig { + max_queue_size: 30000, + max_mem_use: 50 * 1024 * 1024, + } + } +} + /// Block queue status #[derive(Debug)] pub struct BlockQueueInfo { @@ -37,6 +59,12 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, /// Number of blocks being verified pub verifying_queue_size: usize, + /// Configured maximum number of blocks in the queue + pub max_queue_size: usize, + /// Configured maximum number of bytes to use + pub max_mem_use: usize, + /// Heap memory used in bytes + pub mem_used: usize, } impl BlockQueueInfo { @@ -48,7 +76,8 @@ impl BlockQueueInfo { /// Indicates that queue is full pub fn is_full(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > MAX_UNVERIFIED_QUEUE_SIZE + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || + self.mem_used > self.max_mem_use } /// Indicates that queue is empty @@ -68,7 +97,9 @@ pub struct BlockQueue { deleting: Arc, ready_signal: Arc, empty: Arc, - processing: RwLock> + processing: RwLock>, + max_queue_size: usize, + max_mem_use: usize, } struct UnVerifiedBlock { @@ -106,11 +137,9 @@ struct Verification { bad: HashSet, } -const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; - impl BlockQueue { /// Creates a new queue instance. - pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { + pub fn new(config: BlockQueueConfig, engine: Arc>, message_channel: IoChannel) -> BlockQueue { let verification = Arc::new(Mutex::new(Verification::default())); let more_to_verify = Arc::new(Condvar::new()); let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); @@ -133,7 +162,7 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) + BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) .expect("Error starting block verification thread") @@ -149,6 +178,8 @@ impl BlockQueue { deleting: deleting.clone(), processing: RwLock::new(HashSet::new()), empty: empty.clone(), + max_queue_size: config.max_queue_size, + max_mem_use: config.max_mem_use, } } @@ -334,8 +365,26 @@ impl BlockQueue { verified_queue_size: verification.verified.len(), unverified_queue_size: verification.unverified.len(), verifying_queue_size: verification.verifying.len(), + max_queue_size: self.max_queue_size, + max_mem_use: self.max_mem_use, + mem_used: + verification.unverified.heap_size_of_children() + + verification.verifying.heap_size_of_children() + + verification.verified.heap_size_of_children(), + // TODO: https://github.com/servo/heapsize/pull/50 + //+ self.processing.read().unwrap().heap_size_of_children(), } } + + pub fn collect_garbage(&self) { + { + let mut verification = self.verification.lock().unwrap(); + verification.unverified.shrink_to_fit(); + verification.verifying.shrink_to_fit(); + verification.verified.shrink_to_fit(); + } + self.processing.write().unwrap().shrink_to_fit(); + } } impl MayPanic for BlockQueue { @@ -367,7 +416,7 @@ mod tests { fn get_test_queue() -> BlockQueue { let spec = get_test_spec(); let engine = spec.to_engine().unwrap(); - BlockQueue::new(Arc::new(engine), IoChannel::disconnected()) + BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()) } #[test] @@ -375,7 +424,7 @@ mod tests { // TODO better test let spec = Spec::new_test(); let engine = spec.to_engine().unwrap(); - let _ = BlockQueue::new(Arc::new(engine), IoChannel::disconnected()); + let _ = BlockQueue::new(BlockQueueConfig::default(), Arc::new(engine), IoChannel::disconnected()); } #[test] diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index cc9ff56fd..6907369f4 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -23,6 +23,24 @@ use extras::*; use transaction::*; use views::*; +/// Blockchain configuration. +#[derive(Debug)] +pub struct BlockChainConfig { + /// Preferred cache size in bytes. + pub pref_cache_size: usize, + /// Maximum cache size in bytes. + pub max_cache_size: usize, +} + +impl Default for BlockChainConfig { + fn default() -> Self { + BlockChainConfig { + pref_cache_size: 1 << 14, + max_cache_size: 1 << 20, + } + } +} + /// Represents a tree route between `from` block and `to` block: pub struct TreeRoute { /// A vector of hashes of all blocks, ordered from `from` to `to`. @@ -50,7 +68,7 @@ pub struct CacheSize { impl CacheSize { /// Total amount used by the cache. - fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } + pub fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } } /// Information about best block gathered together @@ -220,33 +238,7 @@ const COLLECTION_QUEUE_SIZE: usize = 8; impl BlockChain { /// Create new instance of blockchain from given Genesis - /// - /// ```rust - /// extern crate ethcore_util as util; - /// extern crate ethcore; - /// use std::env; - /// use std::str::FromStr; - /// use ethcore::spec::*; - /// use ethcore::blockchain::*; - /// use ethcore::ethereum; - /// use util::hash::*; - /// use util::uint::*; - /// - /// fn main() { - /// let spec = ethereum::new_frontier(); - /// - /// let mut dir = env::temp_dir(); - /// dir.push(H32::random().hex()); - /// - /// let bc = BlockChain::new(&spec.genesis_block(), &dir); - /// - /// let genesis_hash = "d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"; - /// assert_eq!(bc.genesis_hash(), H256::from_str(genesis_hash).unwrap()); - /// assert!(bc.is_known(&bc.genesis_hash())); - /// assert_eq!(bc.genesis_hash(), bc.block_hash(0).unwrap()); - /// } - /// ``` - pub fn new(genesis: &[u8], path: &Path) -> BlockChain { + pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { // open extras db let mut extras_path = path.to_path_buf(); extras_path.push("extras"); @@ -261,8 +253,8 @@ impl BlockChain { (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: 1 << 14, - max_cache_size: 1 << 20, + pref_cache_size: config.pref_cache_size, + max_cache_size: config.max_cache_size, best_block: RwLock::new(BestBlock::new()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), @@ -536,6 +528,8 @@ impl BlockChain { } /// Returns true if transaction is known. + // TODO: Use me + #[allow(dead_code)] pub fn is_known_transaction(&self, hash: &H256) -> bool { self.query_extras_exist(hash, &self.transaction_addresses) } @@ -556,6 +550,8 @@ impl BlockChain { } /// Get the transactions' log blooms of a block. + // TODO: Use me + #[allow(dead_code)] pub fn log_blooms(&self, hash: &H256) -> Option { self.query_extras(hash, &self.block_logs) } @@ -671,7 +667,7 @@ mod tests { let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0925002c3260b44e44c3edebad1cc442142b03020209df1ab8bb86752edbd2cd7a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a0363659b251bf8b819179874c8cce7b9b983d7f3704cbb58a3b334431f7032871889032d09c281e1236c0c0".from_hex().unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); let genesis_hash = H256::from_str("3caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942").unwrap(); @@ -715,7 +711,7 @@ mod tests { let best_block_hash = H256::from_str("c208f88c9f5bf7e00840439742c12e5226d9752981f3ec0521bdcb6dd08af277").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1); bc.insert_block(&b2); bc.insert_block(&b3a); @@ -794,14 +790,14 @@ mod tests { let temp = RandomTempPath::new(); { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), genesis_hash); bc.insert_block(&b1); assert_eq!(bc.best_block_hash(), b1_hash); } { - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), b1_hash); } } @@ -854,7 +850,7 @@ mod tests { let b1_hash = H256::from_str("f53f268d23a71e85c7d6d83a9504298712b84c1a2ba220441c86eeda0bf0b6e3").unwrap(); let temp = RandomTempPath::new(); - let bc = BlockChain::new(&genesis, temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); bc.insert_block(&b1); let transactions = bc.transactions(&b1_hash).unwrap(); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c3ec4b4d0..333917e4a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -19,7 +19,7 @@ use util::*; use util::panics::*; use rocksdb::{Options, DB, DBCompactionStyle}; -use blockchain::{BlockChain, BlockProvider, CacheSize}; +use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; use header::BlockNumber; @@ -27,14 +27,16 @@ use state::State; use spec::Spec; use engine::Engine; use views::HeaderView; -use block_queue::{BlockQueue, BlockQueueInfo}; +use block_queue::BlockQueue; use service::{NetSyncMessage, SyncMessage}; use env_info::LastHashes; use verification::*; use block::*; use transaction::LocalizedTransaction; use extras::TransactionAddress; -pub use blockchain::TreeRoute; +pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; +pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; + /// Uniquely identifies block. #[derive(Debug, PartialEq, Clone)] @@ -73,7 +75,16 @@ pub enum BlockStatus { Unknown, } -/// Information about the blockchain gthered together. +/// Client configuration. Includes configs for all sub-systems. +#[derive(Debug, Default)] +pub struct ClientConfig { + /// Block queue configuration. + pub queue: BlockQueueConfig, + /// Blockchain configuration. + pub blockchain: BlockChainConfig, +} + +/// Information about the blockchain gathered together. #[derive(Debug)] pub struct BlockChainInfo { /// Blockchain difficulty. @@ -183,14 +194,14 @@ const CLIENT_DB_VER_STR: &'static str = "2.1"; impl Client { /// Create a new client with given spec and DB path. - pub fn new(spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); let path = dir.as_path(); let gb = spec.genesis_block(); - let chain = Arc::new(RwLock::new(BlockChain::new(&gb, path))); + let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); let mut opts = Options::new(); opts.set_max_open_files(256); opts.create_if_missing(true); @@ -223,7 +234,7 @@ impl Client { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } - let block_queue = BlockQueue::new(engine.clone(), message_channel); + let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel); let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); @@ -330,7 +341,7 @@ impl Client { } /// Get info on the cache. - pub fn cache_info(&self) -> CacheSize { + pub fn blockchain_cache_info(&self) -> BlockChainCacheSize { self.chain.read().unwrap().cache_size() } @@ -342,6 +353,7 @@ impl Client { /// Tick the client. pub fn tick(&self) { self.chain.read().unwrap().collect_garbage(); + self.block_queue.read().unwrap().collect_garbage(); } /// Set up the cache behaviour. diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 3d43fd725..17bd52159 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -85,7 +85,7 @@ #[macro_use] extern crate lazy_static; extern crate rustc_serialize; extern crate rocksdb; -extern crate heapsize; +#[macro_use] extern crate heapsize; extern crate crypto; extern crate time; extern crate env_logger; @@ -96,8 +96,6 @@ extern crate crossbeam; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; -pub mod blockchain; -pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -129,6 +127,8 @@ mod substate; mod executive; mod externalities; mod verification; +mod block_queue; +mod blockchain; #[cfg(test)] mod tests; diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 534aab49d..ea80e7c80 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -20,7 +20,7 @@ use util::*; use util::panics::*; use spec::Spec; use error::*; -use client::Client; +use client::{Client, ClientConfig}; /// Message type for external and internal events #[derive(Clone)] @@ -43,14 +43,14 @@ pub struct ClientService { impl ClientService { /// Start the service in a separate thread. - pub fn start(spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { + pub fn start(config: ClientConfig, spec: Spec, net_config: NetworkConfiguration, db_path: &Path) -> Result { let panic_handler = PanicHandler::new_in_arc(); let mut net_service = try!(NetworkService::start(net_config)); panic_handler.forward_from(&net_service); info!("Starting {}", net_service.host_info()); info!("Configured for {} using {} engine", spec.name, spec.engine_name); - let client = try!(Client::new(spec, db_path, net_service.io().channel())); + let client = try!(Client::new(config, spec, db_path, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { client: client.clone() @@ -130,12 +130,13 @@ mod tests { use tests::helpers::*; use util::network::*; use devtools::*; + use client::ClientConfig; #[test] fn it_can_be_started() { let spec = get_test_spec(); let temp_path = RandomTempPath::new(); - let service = ClientService::start(spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); + let service = ClientService::start(ClientConfig::default(), spec, NetworkConfiguration::new_with_port(40456), &temp_path.as_path()); assert!(service.is_ok()); } } diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index af25d1b72..83df81fa2 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client, BlockId}; +use client::{BlockChainClient, Client, ClientConfig, BlockId}; use tests::helpers::*; use common::*; use devtools::*; @@ -22,14 +22,14 @@ use devtools::*; #[test] fn created() { let dir = RandomTempPath::new(); - let client_result = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()); + let client_result = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()); assert!(client_result.is_ok()); } #[test] fn imports_from_empty() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); client.import_verified_blocks(&IoChannel::disconnected()); client.flush_queue(); } @@ -37,7 +37,7 @@ fn imports_from_empty() { #[test] fn imports_good_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let good_block = get_good_dummy_block(); if let Err(_) = client.import_block(good_block) { panic!("error importing block being good by definition"); @@ -52,7 +52,7 @@ fn imports_good_block() { #[test] fn query_none_block() { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let non_existant = client.block_header(BlockId::Number(188)); assert!(non_existant.is_none()); @@ -104,5 +104,5 @@ fn can_collect_garbage() { let client_result = generate_dummy_client(100); let client = client_result.reference(); client.tick(); - assert!(client.cache_info().blocks < 100 * 1024); + assert!(client.blockchain_cache_info().blocks < 100 * 1024); } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 56653e820..510833f4d 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use client::{BlockChainClient, Client}; +use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; -use blockchain::{BlockChain}; +use blockchain::{BlockChain, BlockChainConfig}; use state::*; use rocksdb::*; use evm::{Schedule, Factory}; @@ -135,7 +135,7 @@ pub fn create_test_block_with_data(header: &Header, transactions: &[&SignedTrans pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); let test_spec = get_test_spec(); let test_engine = test_spec.to_engine().unwrap(); let state_root = test_engine.spec().genesis_header().state_root; @@ -173,7 +173,7 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult> { let dir = RandomTempPath::new(); - let client = Client::new(get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); for block in &blocks { if let Err(_) = client.import_block(block.clone()) { panic!("panic importing block which is well-formed"); @@ -190,7 +190,7 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block(block_order, bc.best_block_hash())); } @@ -203,7 +203,7 @@ pub fn generate_dummy_blockchain(block_number: u32) -> GuardedTempResult GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); for block_order in 1..block_number { bc.insert_block(&create_unverifiable_block_with_extra(block_order, bc.best_block_hash(), None)); } @@ -216,7 +216,7 @@ pub fn generate_dummy_blockchain_with_extra(block_number: u32) -> GuardedTempRes pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { let temp = RandomTempPath::new(); - let bc = BlockChain::new(&create_unverifiable_block(0, H256::zero()), temp.as_path()); + let bc = BlockChain::new(BlockChainConfig::default(), &create_unverifiable_block(0, H256::zero()), temp.as_path()); GuardedTempResult:: { _temp: temp, diff --git a/parity/main.rs b/parity/main.rs index e95f38f13..fad2be840 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -31,6 +31,7 @@ extern crate ctrlc; extern crate fdlimit; extern crate daemonize; extern crate time; +extern crate number_prefix; #[cfg(feature = "rpc")] extern crate ethcore_rpc as rpc; @@ -47,10 +48,10 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethcore::blockchain::CacheSize; use ethsync::{EthSync, SyncConfig}; use docopt::Docopt; use daemonize::Daemonize; +use number_prefix::{binary_prefix, Standalone, Prefixed}; const USAGE: &'static str = " Parity. Ethereum Client. @@ -78,6 +79,7 @@ Options: --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in bytes [default: 262144]. + --queue-max-size BYTES Specify the maximum size of memory to use for block queue [default: 52428800]. -j --jsonrpc Enable the JSON-RPC API sever. --jsonrpc-url URL Specify URL for JSON-RPC API server [default: 127.0.0.1:8545]. @@ -105,6 +107,7 @@ struct Args { flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, + flag_queue_max_size: usize, flag_jsonrpc: bool, flag_jsonrpc_url: String, flag_logging: Option, @@ -285,9 +288,12 @@ impl Configuration { sync_config.network_id = spec.network_id(); // Build client - let mut service = ClientService::start(spec, net_settings, &Path::new(&self.path())).unwrap(); + let mut client_config = ClientConfig::default(); + client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; + client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.queue.max_mem_use = self.args.flag_queue_max_size; + let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); - client.configure_cache(self.args.flag_cache_pref_size, self.args.flag_cache_max_size); // Sync let sync = EthSync::register(service.network(), sync_config, client); @@ -331,7 +337,7 @@ fn main() { struct Informant { chain_info: RwLock>, - cache_info: RwLock>, + cache_info: RwLock>, report: RwLock>, } @@ -346,18 +352,26 @@ impl Default for Informant { } impl Informant { + + fn format_bytes(b: usize) -> String { + match binary_prefix(b as f64) { + Standalone(bytes) => format!("{} bytes", bytes), + Prefixed(prefix, n) => format!("{:.0} {}B", n, prefix), + } + } + pub fn tick(&self, client: &Client, sync: &EthSync) { // 5 seconds betwen calls. TODO: calculate this properly. let dur = 5usize; let chain_info = client.chain_info(); let queue_info = client.queue_info(); - let cache_info = client.cache_info(); + let cache_info = client.blockchain_cache_info(); let report = client.report(); let sync_info = sync.status(); - if let (_, &Some(ref last_cache_info), &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { - println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// {} ({}) bl {} ({}) ex ]", + if let (_, _, &Some(ref last_report)) = (self.chain_info.read().unwrap().deref(), self.cache_info.read().unwrap().deref(), self.report.read().unwrap().deref()) { + println!("[ #{} {} ]---[ {} blk/s | {} tx/s | {} gas/s //··· {}/{} peers, #{}, {}+{} queued ···// mem: {} chain, {} queue, {} sync ]", chain_info.best_block_number, chain_info.best_block_hash, (report.blocks_imported - last_report.blocks_imported) / dur, @@ -370,10 +384,9 @@ impl Informant { queue_info.unverified_queue_size, queue_info.verified_queue_size, - cache_info.blocks, - cache_info.blocks as isize - last_cache_info.blocks as isize, - cache_info.block_details, - cache_info.block_details as isize - last_cache_info.block_details as isize + Informant::format_bytes(cache_info.total()), + Informant::format_bytes(queue_info.mem_used), + Informant::format_bytes(sync_info.mem_used), ); } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 6e92184c8..05b190573 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -16,7 +16,7 @@ use util::*; use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId}; -use ethcore::block_queue::BlockQueueInfo; +use ethcore::BlockQueueInfo; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; @@ -242,6 +242,8 @@ impl BlockChainClient for TestBlockChainClient { verified_queue_size: 0, unverified_queue_size: 0, verifying_queue_size: 0, + max_unverified: 0, + mem_used: 0, } } From bed89d0740ffe39000291ae42f60a7778760930a Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Feb 2016 16:32:34 +0100 Subject: [PATCH 4/6] Fixed sync tests --- sync/src/lib.rs | 4 ++-- sync/src/tests/helpers.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 9bf715f07..6f28fc320 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -34,14 +34,14 @@ //! use std::env; //! use std::sync::Arc; //! use util::network::{NetworkService, NetworkConfiguration}; -//! use ethcore::client::Client; +//! use ethcore::client::{Client, ClientConfig}; //! use ethsync::{EthSync, SyncConfig}; //! use ethcore::ethereum; //! //! fn main() { //! let mut service = NetworkService::start(NetworkConfiguration::new()).unwrap(); //! let dir = env::temp_dir(); -//! let client = Client::new(ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); +//! let client = Client::new(ClientConfig::default(), ethereum::new_frontier(), &dir, service.io().channel()).unwrap(); //! EthSync::register(&mut service, SyncConfig::default(), client); //! } //! ``` diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 05b190573..6f4392104 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -15,8 +15,7 @@ // along with Parity. If not, see . use util::*; -use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId}; -use ethcore::BlockQueueInfo; +use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId, BlockQueueInfo}; use ethcore::header::{Header as BlockHeader, BlockNumber}; use ethcore::error::*; use io::SyncIo; @@ -242,7 +241,8 @@ impl BlockChainClient for TestBlockChainClient { verified_queue_size: 0, unverified_queue_size: 0, verifying_queue_size: 0, - max_unverified: 0, + max_queue_size: 0, + max_mem_use: 0, mem_used: 0, } } From 0344f2b4c98373dd8745af4ddb1708a3e1a0fbbf Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Feb 2016 17:14:45 +0100 Subject: [PATCH 5/6] Block queue mem limit test --- ethcore/src/block_queue.rs | 22 ++++++++++++++++++++-- ethcore/src/tests/helpers.rs | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 0f5985f06..879798868 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -30,6 +30,9 @@ use util::panics::*; known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); +const MIN_MEM_LIMIT: usize = 16384; +const MIN_QUEUE_LIMIT: usize = 512; + /// Block queue configuration #[derive(Debug)] pub struct BlockQueueConfig { @@ -178,8 +181,8 @@ impl BlockQueue { deleting: deleting.clone(), processing: RwLock::new(HashSet::new()), empty: empty.clone(), - max_queue_size: config.max_queue_size, - max_mem_use: config.max_mem_use, + max_queue_size: max(config.max_queue_size, MIN_QUEUE_LIMIT), + max_mem_use: max(config.max_mem_use, MIN_MEM_LIMIT), } } @@ -480,4 +483,19 @@ mod tests { assert!(queue.queue_info().is_empty()); } + + #[test] + fn test_mem_limit() { + let spec = get_test_spec(); + let engine = spec.to_engine().unwrap(); + let mut config = BlockQueueConfig::default(); + config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 + let mut queue = BlockQueue::new(config, Arc::new(engine), IoChannel::disconnected()); + assert!(!queue.queue_info().is_full()); + let mut blocks = get_good_dummy_block_seq(50); + for b in blocks.drain(..) { + queue.import_block(b).unwrap(); + } + assert!(queue.queue_info().is_full()); + } } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 510833f4d..808d2e5fb 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -253,6 +253,25 @@ pub fn get_temp_state_in(path: &Path) -> State { State::new(journal_db, U256::from(0u8)) } +pub fn get_good_dummy_block_seq(count: usize) -> Vec { + let test_spec = get_test_spec(); + let test_engine = test_spec.to_engine().unwrap(); + let mut parent = test_engine.spec().genesis_header().hash(); + let mut r = Vec::new(); + for i in 1 .. count + 1 { + let mut block_header = Header::new(); + block_header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap()); + block_header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap()); + block_header.timestamp = i as u64; + block_header.number = i as u64; + block_header.parent_hash = parent; + block_header.state_root = test_engine.spec().genesis_header().state_root; + parent = block_header.hash(); + r.push(create_test_block(&block_header)); + } + r +} + pub fn get_good_dummy_block() -> Bytes { let mut block_header = Header::new(); let test_spec = get_test_spec(); From c139b6bcbbfd7abc51bf437d40a7cc9cc166ab30 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 25 Feb 2016 17:48:23 +0100 Subject: [PATCH 6/6] Fixed json tests build --- ethcore/src/json_tests/chain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index a386e2854..89bd5da2b 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use super::test_common::*; -use client::{BlockChainClient,Client}; +use client::{BlockChainClient, Client, ClientConfig}; use pod_state::*; use block::Block; use ethereum; @@ -53,7 +53,7 @@ pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec { let temp = RandomTempPath::new(); { - let client = Client::new(spec, temp.as_path(), IoChannel::disconnected()).unwrap(); + let client = Client::new(ClientConfig::default(), spec, temp.as_path(), IoChannel::disconnected()).unwrap(); assert_eq!(client.chain_info().best_block_hash, genesis_hash); for (b, is_valid) in blocks.into_iter() { if Block::is_good(&b) {