From 7565625ce042cfc8ce328f9387d8732da48cc4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 1 Mar 2016 22:30:23 +0100 Subject: [PATCH 01/30] Integrating TransactionQueue with client --- Cargo.lock | 19 +++++++++ ethcore/src/client.rs | 7 +++ sync/Cargo.toml | 1 + sync/src/chain.rs | 89 ++++++++++++++++++++++++++++++++++----- sync/src/lib.rs | 14 ++++-- sync/src/tests/helpers.rs | 4 ++ 6 files changed, 120 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e558606eb..845c493fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,6 +125,14 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "deque" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "docopt" version = "0.6.78" @@ -259,6 +267,7 @@ dependencies = [ "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -629,6 +638,16 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.1.54" diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index f2894decb..611561c50 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -123,6 +123,9 @@ pub trait BlockChainClient : Sync + Send { /// Get block total difficulty. fn block_total_difficulty(&self, id: BlockId) -> Option; + /// Get address nonce. + fn nonce(&self, address: &Address) -> U256; + /// Get address code. fn code(&self, address: &Address) -> Option; @@ -445,6 +448,10 @@ impl BlockChainClient for Client { Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) } + fn nonce(&self, address: &Address) -> U256 { + self.state().nonce(address) + } + fn code(&self, address: &Address) -> Option { self.state().code(address) } diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 2ce65ca77..993f07a65 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -17,6 +17,7 @@ time = "0.1.34" rand = "0.3.13" heapsize = "0.3" rustc-serialize = "0.3" +rayon = "0.3.1" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 6a7add27f..fa033813e 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -30,14 +30,17 @@ /// use util::*; +use rayon::prelude::*; use std::mem::{replace}; -use ethcore::views::{HeaderView}; +use ethcore::views::{HeaderView, BlockView}; use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId}; use range_collection::{RangeCollection, ToUsize, FromUsize}; use ethcore::error::*; use ethcore::block::Block; +use ethcore::transaction::SignedTransaction; use io::SyncIo; +use transaction_queue::TransactionQueue; use time; use super::SyncConfig; @@ -209,6 +212,8 @@ pub struct ChainSync { max_download_ahead_blocks: usize, /// Network ID network_id: U256, + /// Transactions Queue + transaction_queue: Mutex, } type RlpResponseResult = Result, PacketDecodeError>; @@ -234,6 +239,7 @@ impl ChainSync { last_send_block_number: 0, max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), network_id: config.network_id, + transaction_queue: Mutex::new(TransactionQueue::new()), } } @@ -249,14 +255,14 @@ impl ChainSync { blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), - mem_used: + mem_used: // TODO: https://github.com/servo/heapsize/pull/50 - // self.downloading_hashes.heap_size_of_children() - //+ self.downloading_bodies.heap_size_of_children() - //+ self.downloading_hashes.heap_size_of_children() - self.headers.heap_size_of_children() - + self.bodies.heap_size_of_children() - + self.peers.heap_size_of_children() + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + self.header_ids.heap_size_of_children(), } } @@ -292,6 +298,7 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; + self.transaction_queue.lock().unwrap().clear(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -913,8 +920,16 @@ impl ChainSync { } } /// Called when peer sends us new transactions - fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: PeerId, _r: &UntrustedRlp) -> Result<(), PacketDecodeError> { - Ok(()) + fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + let chain = io.chain(); + let item_count = r.item_count(); + trace!(target: "sync", "{} -> Transactions ({} entries)", peer_id, item_count); + let fetch_latest_nonce = |a : &Address| chain.nonce(a); + for i in 0..item_count { + let tx: SignedTransaction = try!(r.val_at(i)); + self.transaction_queue.lock().unwrap().add(tx, &fetch_latest_nonce); + } + Ok(()) } /// Send Status message @@ -1242,6 +1257,34 @@ impl ChainSync { } self.last_send_block_number = chain.best_block_number; } + + /// called when block is imported to chain, updates transactions queue + pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], bad: &[H256]) { + fn fetch_transactions(chain: &BlockChainClient, hash: &H256) -> Vec { + let block = chain + .block(BlockId::Hash(hash.clone())) + .expect("Expected in-chain blocks."); + let block = BlockView::new(&block); + block.transactions() + }; + + let chain = io.chain(); + let good = good.par_iter().map(|h| fetch_transactions(chain, h)); + let bad = bad.par_iter().map(|h| fetch_transactions(chain, h)); + + good.for_each(|txs| { + let mut transaction_queue = self.transaction_queue.lock().unwrap(); + transaction_queue.remove_all(&txs); + }); + bad.for_each(|txs| { + // populate sender + for tx in &txs { + let _sender = tx.sender(); + } + let mut transaction_queue = self.transaction_queue.lock().unwrap(); + transaction_queue.add_all(txs, |a| chain.nonce(a)); + }); + } } #[cfg(test)] @@ -1571,6 +1614,32 @@ mod tests { assert!(result.is_ok()); } + #[test] + fn should_add_transactions_to_queue() { + // given + let mut client = TestBlockChainClient::new(); + // client.add_blocks(98, BlocksWith::Uncle); + // client.add_blocks(1, BlocksWith::UncleAndTransaction); + // client.add_blocks(1, BlocksWith::Transaction); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); + + let good_blocks = vec![client.block_hash_delta_minus(2)]; + let bad_blocks = vec![client.block_hash_delta_minus(1)]; + + let mut queue = VecDeque::new(); + let io = TestIo::new(&mut client, &mut queue, None); + + // when + sync.chain_new_blocks(&io, &[], &good_blocks); + assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); + sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); + + // then + let status = sync.transaction_queue.lock().unwrap().status(); + assert_eq!(status.pending, 1); + assert_eq!(status.future, 0); + } + #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 74541660d..44f3f02e0 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -54,6 +54,7 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; +extern crate rayon; #[macro_use] extern crate heapsize; @@ -70,8 +71,7 @@ use io::NetSyncIo; mod chain; mod io; mod range_collection; -// TODO [todr] Made public to suppress dead code warnings -pub mod transaction_queue; +mod transaction_queue; #[cfg(test)] mod tests; @@ -153,8 +153,14 @@ impl NetworkProtocolHandler for EthSync { } fn message(&self, io: &NetworkContext, message: &SyncMessage) { - if let SyncMessage::BlockVerified = *message { - self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); + match *message { + SyncMessage::BlockVerified => { + self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); + }, + SyncMessage::NewChainBlocks { ref good, ref bad } => { + let sync_io = NetSyncIo::new(io, self.chain.deref()); + self.sync.write().unwrap().chain_new_blocks(&sync_io, good, bad); + } } } } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index b788e0c2a..302836920 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -105,6 +105,10 @@ impl BlockChainClient for TestBlockChainClient { Some(U256::zero()) } + fn nonce(&self, _address: &Address) -> U256 { + U256::zero() + } + fn code(&self, _address: &Address) -> Option { unimplemented!(); } From c6934431d12af045364bb9256d45305969ccee67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 25 Feb 2016 11:49:12 +0100 Subject: [PATCH 02/30] Adding test for sync.chain_new_blocks. --- sync/src/chain.rs | 33 ++++++++++---------- sync/src/tests/chain.rs | 51 ++++++++++++++++--------------- sync/src/tests/helpers.rs | 57 +++++++++++++++++++++++++++-------- sync/src/transaction_queue.rs | 1 + 4 files changed, 88 insertions(+), 54 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fa033813e..9edab791a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1425,7 +1425,7 @@ mod tests { #[test] fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let io = TestIo::new(&mut client, &mut queue, None); @@ -1438,7 +1438,7 @@ mod tests { #[test] fn calculates_tree_for_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(15, false); + client.add_blocks(15, BlocksWith::Uncle); let start = client.block_hash_delta_minus(4); let end = client.block_hash_delta_minus(2); @@ -1455,7 +1455,7 @@ mod tests { #[test] fn sends_new_hashes_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1475,7 +1475,7 @@ mod tests { #[test] fn sends_latest_block_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1495,7 +1495,7 @@ mod tests { #[test] fn handles_peer_new_block_mallformed() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let block_data = get_dummy_block(11, client.chain_info().best_block_hash); @@ -1513,7 +1513,7 @@ mod tests { #[test] fn handles_peer_new_block() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); @@ -1531,7 +1531,7 @@ mod tests { #[test] fn handles_peer_new_block_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1547,7 +1547,7 @@ mod tests { #[test] fn handles_peer_new_hashes() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1563,7 +1563,7 @@ mod tests { #[test] fn handles_peer_new_hashes_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1581,7 +1581,7 @@ mod tests { #[test] fn hashes_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1600,7 +1600,7 @@ mod tests { #[test] fn block_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1618,9 +1618,9 @@ mod tests { fn should_add_transactions_to_queue() { // given let mut client = TestBlockChainClient::new(); - // client.add_blocks(98, BlocksWith::Uncle); - // client.add_blocks(1, BlocksWith::UncleAndTransaction); - // client.add_blocks(1, BlocksWith::Transaction); + client.add_blocks(98, BlocksWith::Uncle); + client.add_blocks(1, BlocksWith::UncleAndTransaction); + client.add_blocks(1, BlocksWith::Transaction); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; @@ -1631,6 +1631,7 @@ mod tests { // when sync.chain_new_blocks(&io, &[], &good_blocks); + assert_eq!(sync.transaction_queue.lock().unwrap().status().future, 0); assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); @@ -1643,7 +1644,7 @@ mod tests { #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1667,7 +1668,7 @@ mod tests { #[test] fn returns_requested_block_headers_reverse() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index 1dd9a1e78..78663aa16 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -24,8 +24,8 @@ use super::helpers::*; fn two_peers() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); assert_eq!(net.peer(0).chain.blocks.read().unwrap().deref(), net.peer(1).chain.blocks.read().unwrap().deref()); @@ -35,8 +35,8 @@ fn two_peers() { fn status_after_sync() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync(); let status = net.peer(0).sync.status(); assert_eq!(status.state, SyncState::Idle); @@ -45,8 +45,8 @@ fn status_after_sync() { #[test] fn takes_few_steps() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(100, false); - net.peer_mut(2).chain.add_blocks(100, false); + net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(100, BlocksWith::Uncle); let total_steps = net.sync(); assert!(total_steps < 7); } @@ -56,8 +56,9 @@ fn empty_blocks() { ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { - net.peer_mut(1).chain.add_blocks(5, n % 2 == 0); - net.peer_mut(2).chain.add_blocks(5, n % 2 == 0); + let with = if n % 2 == 0 { BlocksWith::Nothing } else { BlocksWith::Uncle }; + net.peer_mut(1).chain.add_blocks(5, with.clone()); + net.peer_mut(2).chain.add_blocks(5, with); } net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); @@ -68,14 +69,14 @@ fn empty_blocks() { fn forked() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(0).chain.add_blocks(300, false); - net.peer_mut(1).chain.add_blocks(300, false); - net.peer_mut(2).chain.add_blocks(300, false); - net.peer_mut(0).chain.add_blocks(100, true); //fork - net.peer_mut(1).chain.add_blocks(200, false); - net.peer_mut(2).chain.add_blocks(200, false); - net.peer_mut(1).chain.add_blocks(100, false); //fork between 1 and 2 - net.peer_mut(2).chain.add_blocks(10, true); + net.peer_mut(0).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, BlocksWith::Nothing); //fork + net.peer_mut(1).chain.add_blocks(200, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(200, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, BlocksWith::Nothing); // peer 1 has the best chain of 601 blocks let peer1_chain = net.peer(1).chain.numbers.read().unwrap().clone(); net.sync(); @@ -87,8 +88,8 @@ fn forked() { #[test] fn restart() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync_steps(8); @@ -109,8 +110,8 @@ fn status_empty() { #[test] fn status_packet() { let mut net = TestNet::new(2); - net.peer_mut(0).chain.add_blocks(100, false); - net.peer_mut(1).chain.add_blocks(1, false); + net.peer_mut(0).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1, BlocksWith::Uncle); net.start(); @@ -123,10 +124,10 @@ fn status_packet() { #[test] fn propagade_hashes() { let mut net = TestNet::new(6); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, false); + net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -149,10 +150,10 @@ fn propagade_hashes() { #[test] fn propagade_blocks() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, false); + net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -164,7 +165,7 @@ fn propagade_blocks() { #[test] fn restart_on_malformed_block() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.peer_mut(1).chain.corrupt_block(6); net.sync_steps(10); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 302836920..bb980e3f9 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -22,7 +22,7 @@ use io::SyncIo; use chain::ChainSync; use ::SyncConfig; use ethcore::receipt::Receipt; -use ethcore::transaction::LocalizedTransaction; +use ethcore::transaction::{LocalizedTransaction, Transaction, Action}; use ethcore::filter::Filter; use ethcore::log_entry::LocalizedLogEntry; @@ -34,6 +34,14 @@ pub struct TestBlockChainClient { pub difficulty: RwLock, } +#[derive(Clone)] +pub enum BlocksWith { + Nothing, + Uncle, + Transaction, + UncleAndTransaction +} + impl TestBlockChainClient { pub fn new() -> TestBlockChainClient { @@ -44,30 +52,53 @@ impl TestBlockChainClient { last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), }; - client.add_blocks(1, true); // add genesis block + client.add_blocks(1, BlocksWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } - pub fn add_blocks(&mut self, count: usize, empty: bool) { + pub fn add_blocks(&mut self, count: usize, with: BlocksWith) { let len = self.numbers.read().unwrap().len(); for n in len..(len + count) { let mut header = BlockHeader::new(); header.difficulty = From::from(n); header.parent_hash = self.last_hash.read().unwrap().clone(); header.number = n as BlockNumber; - let mut uncles = RlpStream::new_list(if empty {0} else {1}); - if !empty { - let mut uncle_header = BlockHeader::new(); - uncle_header.difficulty = From::from(n); - uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); - uncle_header.number = n as BlockNumber; - uncles.append(&uncle_header); - header.uncles_hash = uncles.as_raw().sha3(); - } + let uncles = match with { + BlocksWith::Uncle | BlocksWith::UncleAndTransaction => { + let mut uncles = RlpStream::new_list(1); + let mut uncle_header = BlockHeader::new(); + uncle_header.difficulty = From::from(n); + uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); + uncle_header.number = n as BlockNumber; + uncles.append(&uncle_header); + header.uncles_hash = uncles.as_raw().sha3(); + uncles + }, + _ => RlpStream::new_list(0) + }; + let txs = match with { + BlocksWith::Transaction | BlocksWith::UncleAndTransaction => { + let mut txs = RlpStream::new_list(1); + let keypair = KeyPair::create().unwrap(); + let tx = Transaction { + action: Action::Create, + value: U256::from(100), + data: "3331600055".from_hex().unwrap(), + gas: U256::from(100_000), + gas_price: U256::one(), + nonce: U256::one() + }; + let signed_tx = tx.sign(&keypair.secret()); + txs.append(&signed_tx); + txs.out() + }, + _ => rlp::NULL_RLP.to_vec() + }; + let mut rlp = RlpStream::new_list(3); rlp.append(&header); - rlp.append_raw(&rlp::NULL_RLP, 1); + rlp.append_raw(&txs, 1); rlp.append_raw(uncles.as_raw(), 1); self.import_block(rlp.as_raw().to_vec()).unwrap(); } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 341607afe..fa6026477 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -346,6 +346,7 @@ impl TransactionQueue { return; } else if next_nonce > nonce { // Droping transaction + trace!(target: "sync", "Dropping transaction with nonce: {} - expecting: {}", nonce, next_nonce); return; } From c75737bcf0318a51f7bc19ab23d78e630481adf6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 17:04:44 +0100 Subject: [PATCH 03/30] Add ancestry iterator. --- ethcore/src/blockchain/blockchain.rs | 52 ++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 185bcaad3..6e5d92a45 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -227,6 +227,23 @@ impl BlockProvider for BlockChain { const COLLECTION_QUEUE_SIZE: usize = 8; +pub struct AncestryIter<'a> { + current: H256, + chain: &'a BlockChain, +} +impl<'a> Iterator for AncestryIter<'a> { + type Item = H256; + fn next(&mut self) -> Option { + if self.current.is_zero() { + Option::None + } else { + let n = self.chain.block_details(&self.current).unwrap().parent; + self.current = n; + Some(self.current.clone()) + } + } +} + impl BlockChain { /// Create new instance of blockchain from given Genesis pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { @@ -473,9 +490,40 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } + pub fn ancestry_iter(&self, first: H256) -> AncestryIter { + AncestryIter { + current: first, + chain: &self, + } + } + /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, _parent: &H256) -> Vec
{ - // TODO + pub fn find_uncle_headers(&self, parent: &H256) -> Vec
{ + let uncle_generations = 6usize; +/* + { + // Find great-uncles (or second-cousins or whatever they are) - children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. + clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); + h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); + auto p = m_previousBlock.parentHash(); + for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parent) + { + auto us = _bc.details(p).children; + assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! + for (auto const& u: us) + if (!excluded.count(u)) // ignore any uncles/mainline blocks that we know about. + { + uncleBlockHeaders.push_back(_bc.info(u)); + unclesData.appendRaw(_bc.headerData(u)); + ++unclesCount; + if (unclesCount == 2) + break; + } + } + } +*/ + let _excluded = self.ancestry_iter(parent.clone()).take(uncle_generations).collect::>(); + Vec::new() } From 671965d44fdf60547d85cd0689f62da46321ea4d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 17:31:42 +0100 Subject: [PATCH 04/30] Test for ancestry. --- ethcore/src/blockchain/blockchain.rs | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6e5d92a45..d40a34b0c 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -231,15 +231,16 @@ pub struct AncestryIter<'a> { current: H256, chain: &'a BlockChain, } + impl<'a> Iterator for AncestryIter<'a> { type Item = H256; fn next(&mut self) -> Option { if self.current.is_zero() { Option::None } else { - let n = self.chain.block_details(&self.current).unwrap().parent; - self.current = n; - Some(self.current.clone()) + let mut n = self.chain.block_details(&self.current).unwrap().parent; + mem::swap(&mut self.current, &mut n); + Some(n) } } } @@ -857,6 +858,28 @@ mod tests { assert_eq!(bc.block_hash(2), None); } + #[test] + fn check_ancestry_iter() { + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let genesis_hash = BlockView::new(&genesis).header_view().sha3(); + + let temp = RandomTempPath::new(); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); + + let mut block_hashes = vec![genesis_hash.clone()]; + for _ in 0..10 { + let block = canon_chain.generate(&mut finalizer).unwrap(); + block_hashes.push(BlockView::new(&block).header_view().sha3()); + bc.insert_block(&block, vec![]); + } + + block_hashes.reverse(); + + assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).collect::>(), block_hashes) + } + #[test] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { From 42df98450c3c2edcf9c1898d20c23cb012928b32 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 18:05:47 +0100 Subject: [PATCH 05/30] Include uncles in exclused. --- ethcore/src/blockchain/blockchain.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index d40a34b0c..6779051e2 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -78,7 +78,7 @@ pub trait BlockProvider { } /// Get a list of uncles for a given block. - /// Returns None if block deos not exist. + /// Returns None if block does not exist. fn uncles(&self, hash: &H256) -> Option> { self.block(hash).map(|bytes| BlockView::new(&bytes).uncles()) } @@ -491,7 +491,7 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } - pub fn ancestry_iter(&self, first: H256) -> AncestryIter { + pub fn ancestry_iter(&self, first: H256) -> Option { AncestryIter { current: first, chain: &self, @@ -503,7 +503,8 @@ impl BlockChain { let uncle_generations = 6usize; /* { - // Find great-uncles (or second-cousins or whatever they are) - children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. + // Find great-uncles (or second-cousins or whatever they are) - + // children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); auto p = m_previousBlock.parentHash(); @@ -523,7 +524,11 @@ impl BlockChain { } } */ - let _excluded = self.ancestry_iter(parent.clone()).take(uncle_generations).collect::>(); + let _excluded = self + .ancestry_iter(parent.clone()) + .take(uncle_generations) + .flat_map(|h| self.uncle_hashes(&h).iter().chain(&[h])) + .collect::>(); Vec::new() } From 877270c35fe68adb967a5d0714d5c3e15eaf91fc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 18:32:54 +0100 Subject: [PATCH 06/30] Fixes. --- ethcore/src/blockchain/blockchain.rs | 31 +++++++++++++++++----------- ethcore/src/client.rs | 4 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6779051e2..6b0939df5 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -491,15 +491,20 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } + /// Iterator that lists `first` and then all of `first`'s ancestors, by hash. pub fn ancestry_iter(&self, first: H256) -> Option { - AncestryIter { - current: first, - chain: &self, + if self.is_known(&first) { + Some(AncestryIter { + current: first, + chain: &self, + }) + } else { + None } } /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, parent: &H256) -> Vec
{ + pub fn find_uncle_headers(&self, parent: &H256) -> Option> { let uncle_generations = 6usize; /* { @@ -524,13 +529,15 @@ impl BlockChain { } } */ - let _excluded = self - .ancestry_iter(parent.clone()) - .take(uncle_generations) - .flat_map(|h| self.uncle_hashes(&h).iter().chain(&[h])) - .collect::>(); - - Vec::new() + if !self.is_known(parent) { return None; } + let mut _excluded = HashSet::new(); + for a in self.ancestry_iter(parent.clone()).unwrap().take(uncle_generations) { + for u in self.uncle_hashes(&a).unwrap().into_iter() { + _excluded.insert(u); + } + _excluded.insert(a); + } + None } /// Get inserted block info which is critical to preapre extras updates. @@ -882,7 +889,7 @@ mod tests { block_hashes.reverse(); - assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).collect::>(), block_hashes) + assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).unwrap().collect::>(), block_hashes) } #[test] diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index cc5e23869..105090580 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -200,7 +200,7 @@ pub struct Client { extra_data: RwLock, } -const HISTORY: u64 = 1000; +const HISTORY: u64 = 2000000; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { @@ -457,7 +457,7 @@ impl Client { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h).into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. From 039c0056bc0f58bfe33860c7a4aab7b9aff57bbc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 19:38:00 +0100 Subject: [PATCH 07/30] Uncle inclusion in block authoring. Still need tests. --- ethcore/src/blockchain/blockchain.rs | 46 +++++++++------------------- ethcore/src/client.rs | 2 +- ethcore/src/engine.rs | 2 ++ ethcore/src/verification.rs | 4 +-- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6b0939df5..4cf2d96c5 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -503,41 +503,23 @@ impl BlockChain { } } - /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, parent: &H256) -> Option> { - let uncle_generations = 6usize; -/* - { - // Find great-uncles (or second-cousins or whatever they are) - - // children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. - clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); - h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); - auto p = m_previousBlock.parentHash(); - for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parent) - { - auto us = _bc.details(p).children; - assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! - for (auto const& u: us) - if (!excluded.count(u)) // ignore any uncles/mainline blocks that we know about. - { - uncleBlockHeaders.push_back(_bc.info(u)); - unclesData.appendRaw(_bc.headerData(u)); - ++unclesCount; - if (unclesCount == 2) - break; - } - } - } -*/ + /// Given a block's `parent`, find every block header which represents a valid possible uncle. + pub fn find_uncle_headers(&self, parent: &H256, uncle_generations: usize) -> Option> { if !self.is_known(parent) { return None; } - let mut _excluded = HashSet::new(); + + let mut excluded = HashSet::new(); for a in self.ancestry_iter(parent.clone()).unwrap().take(uncle_generations) { - for u in self.uncle_hashes(&a).unwrap().into_iter() { - _excluded.insert(u); - } - _excluded.insert(a); + excluded.extend(self.uncle_hashes(&a).unwrap().into_iter()); + excluded.insert(a); } - None + + let mut ret = Vec::new(); + for a in self.ancestry_iter(parent.clone()).unwrap().skip(1).take(uncle_generations) { + ret.extend(self.block_details(&a).unwrap().children.iter() + .filter_map(|h| if excluded.contains(h) { None } else { self.block_header(h) }) + ); + } + Some(ret) } /// Get inserted block info which is critical to preapre extras updates. diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 105090580..4ddb40477 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -457,7 +457,7 @@ impl Client { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index d607ce2e2..83e1986fd 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -47,6 +47,8 @@ pub trait Engine : Sync + Send { fn maximum_extra_data_size(&self) -> usize { decode(&self.spec().engine_params.get("maximumExtraDataSize").unwrap()) } /// Maximum number of uncles a block is allowed to declare. fn maximum_uncle_count(&self) -> usize { 2 } + /// The number of generations back that uncles can be. + fn maximum_uncle_age(&self) -> usize { 6 } /// The nonce with which accounts begin. fn account_start_nonce(&self) -> U256 { decode(&self.spec().engine_params.get("accountStartNonce").unwrap()) } diff --git a/ethcore/src/verification.rs b/ethcore/src/verification.rs index f52e2e1e4..23b850f55 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification.rs @@ -94,7 +94,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, b excluded.insert(header.hash()); let mut hash = header.parent_hash.clone(); excluded.insert(hash.clone()); - for _ in 0..6 { + for _ in 0..engine.maximum_uncle_age() { match bc.block_details(&hash) { Some(details) => { excluded.insert(details.parent.clone()); @@ -121,7 +121,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, b // (8 Invalid) let depth = if header.number > uncle.number { header.number - uncle.number } else { 0 }; - if depth > 6 { + if depth > engine.maximum_uncle_age() as u64 { return Err(From::from(BlockError::UncleTooOld(OutOfBounds { min: Some(header.number - depth), max: Some(header.number - 1), found: uncle.number }))); } else if depth < 1 { From 6933bb971ba8947ead271ec8825452fc3fd7ae20 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 23:41:15 +0100 Subject: [PATCH 08/30] Test. --- ethcore/src/blockchain/blockchain.rs | 38 ++++++++++++++++++++++++++++ ethcore/src/header.rs | 21 ++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 4cf2d96c5..394c16e85 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -874,6 +874,44 @@ mod tests { assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).unwrap().collect::>(), block_hashes) } + #[test] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] + fn test_find_uncles() { + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let b1b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b1a = canon_chain.generate(&mut finalizer).unwrap(); + let b2b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b2a = canon_chain.generate(&mut finalizer).unwrap(); + let b3b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b3a = canon_chain.generate(&mut finalizer).unwrap(); + let b4b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b4a = canon_chain.generate(&mut finalizer).unwrap(); + let b5b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b5a = canon_chain.generate(&mut finalizer).unwrap(); + + let temp = RandomTempPath::new(); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); + bc.insert_block(&b1a, vec![]); + bc.insert_block(&b1b, vec![]); + bc.insert_block(&b2a, vec![]); + bc.insert_block(&b2b, vec![]); + bc.insert_block(&b3a, vec![]); + bc.insert_block(&b3b, vec![]); + bc.insert_block(&b4a, vec![]); + bc.insert_block(&b4b, vec![]); + bc.insert_block(&b5a, vec![]); + bc.insert_block(&b5b, vec![]); + + assert_eq!( + [&b4b, &b3b, &b2b].iter().map(|b| BlockView::new(b).header()).collect::>(), + bc.find_uncle_headers(&BlockView::new(&b4a).header_view().sha3(), 3).unwrap() + ); + + // TODO: insert block that already includes one of them as an uncle to check it's not allowed. + } + #[test] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index cc02d84db..1e1a54d57 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -29,7 +29,7 @@ pub type BlockNumber = u64; /// which is non-specific. /// /// Doesn't do all that much on its own. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq)] pub struct Header { // TODO: make all private. /// Parent hash. @@ -70,6 +70,25 @@ pub struct Header { pub bare_hash: RefCell>, } +impl PartialEq for Header { + fn eq(&self, c: &Header) -> bool { + self.parent_hash == c.parent_hash && + self.timestamp == c.timestamp && + self.number == c.number && + self.author == c.author && + self.transactions_root == c.transactions_root && + self.uncles_hash == c.uncles_hash && + self.extra_data == c.extra_data && + self.state_root == c.state_root && + self.receipts_root == c.receipts_root && + self.log_bloom == c.log_bloom && + self.gas_used == c.gas_used && + self.gas_limit == c.gas_limit && + self.difficulty == c.difficulty && + self.seal == c.seal + } +} + impl Default for Header { fn default() -> Self { Header { From 3daa4c6497f48915ad3d156836c24638d1592cbc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 11:39:00 +0100 Subject: [PATCH 09/30] Fix max uncles. --- ethcore/src/block.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index f5788baba..d05ce51b9 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -220,7 +220,7 @@ impl<'x> OpenBlock<'x> { /// NOTE Will check chain constraints and the uncle number but will NOT check /// that the header itself is actually valid. pub fn push_uncle(&mut self, valid_uncle_header: Header) -> Result<(), BlockError> { - if self.block.base.uncles.len() >= self.engine.maximum_uncle_count() { + if self.block.base.uncles.len() > self.engine.maximum_uncle_count() { return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len()})); } // TODO: check number From df77f51bccf1cd396154207e0084147b26bf9f93 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 11:47:24 +0100 Subject: [PATCH 10/30] History to 30 to pass tests. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 4ddb40477..bcddec85a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -200,7 +200,7 @@ pub struct Client { extra_data: RwLock, } -const HISTORY: u64 = 2000000; +const HISTORY: u64 = 30; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { From bcaed67eaa0257c8998925287fd2dbc40df12698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 4 Mar 2016 16:48:10 +0100 Subject: [PATCH 11/30] Swapping order of inserting block to chain and commiting to DB to avoid race conditions --- ethcore/src/client.rs | 12 +++++++----- sync/src/transaction_queue.rs | 4 ---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index d442a3d88..fdcd6c057 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -329,18 +329,14 @@ impl Client { bad_blocks.insert(header.hash()); continue; } - let closed_block = self.check_and_close_block(&block); if let Err(_) = closed_block { bad_blocks.insert(header.hash()); break; } - - // Insert block - let closed_block = closed_block.unwrap(); - self.chain.write().unwrap().insert_block(&block.bytes, closed_block.block().receipts().clone()); good_blocks.push(header.hash()); + // Are we committing an era? let ancient = if header.number() >= HISTORY { let n = header.number() - HISTORY; let chain = self.chain.read().unwrap(); @@ -350,10 +346,16 @@ impl Client { }; // Commit results + let closed_block = closed_block.unwrap(); + let receipts = closed_block.block().receipts().clone(); closed_block.drain() .commit(header.number(), &header.hash(), ancient) .expect("State DB commit failed."); + // And update the chain + self.chain.write().unwrap() + .insert_block(&block.bytes, receipts); + self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index f551fe435..83665dfda 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -240,7 +240,6 @@ impl TransactionQueue { let sender = transaction.sender(); let nonce = transaction.nonce(); - println!("Removing tx: {:?}", transaction.transaction); // Remove from future self.future.drop(&sender, &nonce); @@ -266,7 +265,6 @@ impl TransactionQueue { // Goes to future or is removed let order = self.current.drop(&sender, &k).unwrap(); if k >= current_nonce { - println!("Moving to future: {:?}", order); self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); } else { self.by_hash.remove(&order.hash); @@ -310,7 +308,6 @@ impl TransactionQueue { // remove also from priority and hash self.future.by_priority.remove(&order); // Put to current - println!("Moved: {:?}", order); let order = order.update_height(current_nonce.clone(), first_nonce); self.current.insert(address.clone(), current_nonce, order); current_nonce = current_nonce + U256::one(); @@ -331,7 +328,6 @@ impl TransactionQueue { .cloned() .map_or_else(|| fetch_nonce(&address), |n| n + U256::one()); - println!("Expected next: {:?}, got: {:?}", next_nonce, nonce); // Check height if nonce > next_nonce { let order = TransactionOrder::for_transaction(&tx, next_nonce); From 3fa1776ecf43cb639c5ce688e85a5de1dc17c0f1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 19:11:44 +0100 Subject: [PATCH 12/30] Fixed sync stalling on a new block arriving while body request is pending --- sync/src/chain.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fd690e790..0b985f217 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -475,7 +475,7 @@ impl ChainSync { peer.latest_number = Some(header.number()); } // TODO: Decompose block and add to self.headers and self.bodies instead - if header.number == From::from(self.current_base_block() + 1) { + if header.number <= From::from(self.current_base_block() + 1) { match io.chain().import_block(block_rlp.as_raw().to_vec()) { Err(Error::Import(ImportError::AlreadyInChain)) => { trace!(target: "sync", "New block already in chain {:?}", h); @@ -484,7 +484,10 @@ impl ChainSync { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { - self.last_imported_block = Some(header.number); + if self.current_base_block() < header.number { + self.last_imported_block = Some(header.number); + self.remove_downloaded_blocks(header.number); + } trace!(target: "sync", "New block queued {:?}", h); }, Err(Error::Block(BlockError::UnknownParent(p))) => { From ba67b67ff34a2143be23ff040b54269b78aa7390 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 20:19:36 +0100 Subject: [PATCH 13/30] JournalDB can now operate in "archive" mode. --- util/src/journaldb.rs | 96 +++++++++++++++++++++++++++++++------------ util/src/overlaydb.rs | 2 +- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 5e6ca47c2..cb49134ee 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -25,7 +25,10 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; use std::env; /// Implementation of the HashDB trait for a disk-backed database with a memory overlay -/// and latent-removal semantics. +/// and, possibly, latent-removal semantics. +/// +/// If `counters` is `None`, then it behaves exactly like OverlayDB. If not it behaves +/// differently: /// /// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to /// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect @@ -34,7 +37,7 @@ use std::env; pub struct JournalDB { overlay: MemoryDB, backing: Arc, - counters: Arc>>, + counters: Option>>>, } impl Clone for JournalDB { @@ -48,10 +51,11 @@ impl Clone for JournalDB { } // all keys must be at least 12 bytes -const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION: u32 = 3; +const DB_VERSION : u32 = 3; +const DB_VERSION_NO_JOURNAL : u32 = 3 + 256; const PADDING : [u8; 10] = [ 0u8; 10 ]; @@ -59,25 +63,38 @@ impl JournalDB { /// Create a new instance from file pub fn new(path: &str) -> JournalDB { + Self::from_prefs(path, true) + } + + /// Create a new instance from file + pub fn from_prefs(path: &str, prefer_journal: bool) -> JournalDB { let opts = DatabaseConfig { prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix }; let backing = Database::open(&opts, path).unwrap_or_else(|e| { panic!("Error opening state db: {}", e); }); + let with_journal; if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {}, + Ok(Some(DB_VERSION)) => { with_journal = true; }, + Ok(Some(DB_VERSION_NO_JOURNAL)) => { with_journal = false; }, v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) } } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); + backing.put(&VERSION_KEY, &encode(&(if prefer_journal { DB_VERSION } else { DB_VERSION_NO_JOURNAL }))).expect("Error writing version to database"); + with_journal = prefer_journal; } - let counters = JournalDB::read_counters(&backing); + + let counters = if with_journal { + Some(Arc::new(RwLock::new(JournalDB::read_counters(&backing)))) + } else { + None + }; JournalDB { overlay: MemoryDB::new(), backing: Arc::new(backing), - counters: Arc::new(RwLock::new(counters)), + counters: counters, } } @@ -94,9 +111,48 @@ impl JournalDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } + /// Commit all recent insert operations. + pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let have_counters = self.counters.is_some(); + if have_counters { + self.commit_with_counters(now, id, end) + } else { + self.commit_without_counters() + } + } + + /// Drain the overlay and place it into a batch for the DB. + fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> (usize, usize) { + let mut ret = 0usize; + let mut deletes = 0usize; + for i in overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc > 0 { + assert!(rc == 1); + batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); + ret += 1; + } + if rc < 0 { + assert!(rc == -1); + ret += 1; + deletes += 1; + } + } + (ret, deletes) + } + + /// Just commit the overlay into the backing DB. + fn commit_without_counters(&mut self) -> Result { + let batch = DBTransaction::new(); + let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); + try!(self.backing.write(batch)); + Ok(ret as u32) + + } + /// Commit all recent insert operations and historical removals from the old era /// to the backing database. - pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn commit_with_counters(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -122,8 +178,8 @@ impl JournalDB { // record new commit's details. debug!("commit: #{} ({}), end era: {:?}", now, id, end); + let mut counters = self.counters.as_ref().unwrap().write().unwrap(); let batch = DBTransaction::new(); - let mut counters = self.counters.write().unwrap(); { let mut index = 0usize; let mut last; @@ -196,25 +252,11 @@ impl JournalDB { } // Commit overlay insertions - let mut ret = 0u32; - let mut deletes = 0usize; - for i in self.overlay.drain().into_iter() { - let (key, (value, rc)) = i; - if rc > 0 { - assert!(rc == 1); - batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); - ret += 1; - } - if rc < 0 { - assert!(rc == -1); - ret += 1; - deletes += 1; - } - } + let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); debug!("commit: Deleted {} nodes", deletes); - Ok(ret) + Ok(ret as u32) } diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index f4ed2d5d6..3c80f4148 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -146,7 +146,7 @@ impl OverlayDB { }) } - /// Get the refs and value of the given key. + /// Put the refs and value of the given key, possibly deleting it from the db. fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool { if payload.1 > 0 { let mut s = RlpStream::new_list(2); From bbbaffbc531571894b34f5c5a0de1e7eda453c60 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 21:06:28 +0100 Subject: [PATCH 14/30] "--archive" option for disabling the journal DB Fixes #579 --- ethcore/src/blockchain/blockchain.rs | 3 +++ ethcore/src/client.rs | 5 +++-- parity/main.rs | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index f412a8240..e79f1668c 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -40,6 +40,8 @@ pub struct BlockChainConfig { pub pref_cache_size: usize, /// Maximum cache size in bytes. pub max_cache_size: usize, + /// Prefer journal rather than archive. + pub prefer_journal: bool, } impl Default for BlockChainConfig { @@ -47,6 +49,7 @@ impl Default for BlockChainConfig { BlockChainConfig { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, + prefer_journal: false, } } } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 422f1eaa2..87f2c9e96 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -212,7 +212,8 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); + dir.push(format!("v{}-sec-pruned-{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "journal" } else { "archive" })); + let pj = config.blockchain.prefer_journal; let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); @@ -220,7 +221,7 @@ impl Client { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::new(state_path.to_str().unwrap()); + let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), pj); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/parity/main.rs b/parity/main.rs index b991f36cd..f1a11229f 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -65,6 +65,7 @@ Usage: Options: --chain CHAIN Specify the blockchain type. CHAIN may be either a JSON chain specification file or frontier, mainnet, morden, or testnet [default: frontier]. + --archive Client should not prune the state/storage trie. -d --db-path PATH Specify the database & configuration directory path [default: $HOME/.parity] --keys-path PATH Specify the path for JSON key files to be found [default: $HOME/.web3/keys] @@ -102,6 +103,7 @@ struct Args { flag_chain: String, flag_db_path: String, flag_keys_path: String, + flag_archive: bool, flag_no_bootstrap: bool, flag_listen_address: String, flag_public_address: Option, @@ -311,6 +313,7 @@ impl Configuration { let mut client_config = ClientConfig::default(); client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.blockchain.prefer_journal = !self.args.flag_archive; client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); From bc018faedcffdcea398cee7848a4e2d3a64ce78a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 21:17:42 +0100 Subject: [PATCH 15/30] Avoid forcing a resync for the pre-existing journaldbs. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 87f2c9e96..4c9e76c76 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -212,7 +212,7 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned-{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "journal" } else { "archive" })); + dir.push(format!("v{}-sec-pruned{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "" } else { "-archive" })); let pj = config.blockchain.prefer_journal; let path = dir.as_path(); let gb = spec.genesis_block(); From f028ff1d40f95c5a5f8f0ab3cf85a9df5873b57c Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 21:52:03 +0100 Subject: [PATCH 16/30] Use same BlockChainInfo for propagation --- sync/src/chain.rs | 56 +++++++++++++++++---------------------- sync/src/tests/helpers.rs | 2 +- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fd690e790..cce8b12cf 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -33,7 +33,7 @@ use util::*; use std::mem::{replace}; use ethcore::views::{HeaderView}; use ethcore::header::{BlockNumber, Header as BlockHeader}; -use ethcore::client::{BlockChainClient, BlockStatus, BlockId}; +use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo}; use range_collection::{RangeCollection, ToUsize, FromUsize}; use ethcore::error::*; use ethcore::block::Block; @@ -1156,9 +1156,7 @@ impl ChainSync { } /// returns peer ids that have less blocks than our chain - fn get_lagging_peers(&mut self, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { - let chain = io.chain(); - let chain_info = chain.chain_info(); + fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { let latest_hash = chain_info.best_block_hash; let latest_number = chain_info.best_block_number; self.peers.iter_mut().filter_map(|(&id, ref mut peer_info)| @@ -1177,9 +1175,9 @@ impl ChainSync { } /// propagates latest block to lagging peers - fn propagate_blocks(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { + fn propagate_blocks(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> usize { let updated_peers = { - let lagging_peers = self.get_lagging_peers(io); + let lagging_peers = self.get_lagging_peers(chain_info, io); // sqrt(x)/x scaled to max u32 let fraction = (self.peers.len() as f64).powf(-0.5).mul(u32::max_value() as f64).round() as u32; @@ -1196,30 +1194,30 @@ impl ChainSync { for peer_id in updated_peers { let rlp = ChainSync::create_latest_block_rlp(io.chain()); self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); - self.peers.get_mut(&peer_id).unwrap().latest_hash = local_best.clone(); - self.peers.get_mut(&peer_id).unwrap().latest_number = Some(best_number); + self.peers.get_mut(&peer_id).unwrap().latest_hash = chain_info.best_block_hash.clone(); + self.peers.get_mut(&peer_id).unwrap().latest_number = Some(chain_info.best_block_number); sent = sent + 1; } sent } /// propagates new known hashes to all peers - fn propagate_new_hashes(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { - let updated_peers = self.get_lagging_peers(io); + fn propagate_new_hashes(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> usize { + let updated_peers = self.get_lagging_peers(chain_info, io); let mut sent = 0; - let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(local_best.clone())).unwrap()).parent_hash(); + let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(chain_info.best_block_hash.clone())).unwrap()).parent_hash(); for (peer_id, peer_number) in updated_peers { let mut peer_best = self.peers.get(&peer_id).unwrap().latest_hash.clone(); - if best_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { + if chain_info.best_block_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { // If we think peer is too far behind just send one latest hash peer_best = last_parent.clone(); } - sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &local_best) { + sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &chain_info.best_block_hash) { Some(rlp) => { { let peer = self.peers.get_mut(&peer_id).unwrap(); - peer.latest_hash = local_best.clone(); - peer.latest_number = Some(best_number); + peer.latest_hash = chain_info.best_block_hash.clone(); + peer.latest_number = Some(chain_info.best_block_number); } self.send_packet(io, peer_id, NEW_BLOCK_HASHES_PACKET, rlp); 1 @@ -1239,8 +1237,8 @@ impl ChainSync { pub fn chain_blocks_verified(&mut self, io: &mut SyncIo) { let chain = io.chain().chain_info(); if (((chain.best_block_number as i64) - (self.last_send_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let blocks = self.propagate_blocks(&chain.best_block_hash, chain.best_block_number, io); - let hashes = self.propagate_new_hashes(&chain.best_block_hash, chain.best_block_number, io); + let blocks = self.propagate_blocks(&chain, io); + let hashes = self.propagate_new_hashes(&chain, io); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); } @@ -1390,9 +1388,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); + let chain_info = client.chain_info(); let io = TestIo::new(&mut client, &mut queue, None); - let lagging_peers = sync.get_lagging_peers(&io); + let lagging_peers = sync.get_lagging_peers(&chain_info, &io); assert_eq!(1, lagging_peers.len()) } @@ -1420,11 +1419,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagate_new_hashes(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_new_hashes(&chain_info, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1440,11 +1438,9 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - - let peer_count = sync.propagate_blocks(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_blocks(&chain_info, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1546,11 +1542,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagate_new_hashes(&best_hash, best_number, &mut io); + sync.propagate_new_hashes(&chain_info, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_hashes(&mut io, 0, &UntrustedRlp::new(&data)); @@ -1565,11 +1560,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagate_blocks(&best_hash, best_number, &mut io); + sync.propagate_blocks(&chain_info, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &UntrustedRlp::new(&data)); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index e9c5f0edc..e170a4a85 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -105,7 +105,7 @@ impl BlockChainClient for TestBlockChainClient { Some(U256::zero()) } - fn block_hash(&self, id: BlockId) -> Option { + fn block_hash(&self, _id: BlockId) -> Option { unimplemented!(); } From 182aec2f9463f0c96b6bec1022e5328935785248 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 22:01:36 +0100 Subject: [PATCH 17/30] Fixed potential deadlock on startup --- util/src/network/host.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 42e8ff93d..f2cc9fe48 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -400,7 +400,8 @@ impl Host where Message: Send + Sync + Clone { // public_endpoint in host info contains local adderss at this point let listen_address = self.info.read().unwrap().public_endpoint.address.clone(); let udp_port = self.info.read().unwrap().config.udp_port.unwrap_or(listen_address.port()); - let public_endpoint = match self.info.read().unwrap().config.public_address { + let public_address = self.info.read().unwrap().config.public_address.clone(); + let public_endpoint = match public_address { None => { let public_address = select_public_address(listen_address.port()); let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; From 559e01ea84beb7ff46c67868c69d6e817ba1f986 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 22:54:59 +0100 Subject: [PATCH 18/30] Review remarks resolved. --- ethcore/src/client.rs | 19 +++++++++++++++---- parity/main.rs | 2 +- util/src/journaldb.rs | 9 ++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 4c9e76c76..064b749a8 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -76,12 +76,24 @@ pub enum BlockStatus { } /// Client configuration. Includes configs for all sub-systems. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct ClientConfig { /// Block queue configuration. pub queue: BlockQueueConfig, /// Blockchain configuration. pub blockchain: BlockChainConfig, + /// Prefer journal rather than archive. + pub prefer_journal: bool, +} + +impl Default for ClientConfig { + fn default() -> ClientConfig { + ClientConfig { + queue: Default::default(), + blockchain: Default::default(), + prefer_journal: false, + } + } } /// Information about the blockchain gathered together. @@ -212,8 +224,7 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "" } else { "-archive" })); - let pj = config.blockchain.prefer_journal; + dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, if config.prefer_journal { "pruned" } else { "archive" })); let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); @@ -221,7 +232,7 @@ impl Client { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), pj); + let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/parity/main.rs b/parity/main.rs index f1a11229f..3f4243a0a 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -313,7 +313,7 @@ impl Configuration { let mut client_config = ClientConfig::default(); client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; - client_config.blockchain.prefer_journal = !self.args.flag_archive; + client_config.prefer_journal = !self.args.flag_archive; client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index cb49134ee..b20934397 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -147,7 +147,6 @@ impl JournalDB { let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); Ok(ret as u32) - } /// Commit all recent insert operations and historical removals from the old era @@ -177,7 +176,7 @@ impl JournalDB { // and the key is safe to delete. // record new commit's details. - debug!("commit: #{} ({}), end era: {:?}", now, id, end); + trace!("commit: #{} ({}), end era: {:?}", now, id, end); let mut counters = self.counters.as_ref().unwrap().write().unwrap(); let batch = DBTransaction::new(); { @@ -248,14 +247,14 @@ impl JournalDB { try!(batch.delete(&h)); deletes += 1; } - debug!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); + trace!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); } // Commit overlay insertions let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); - debug!("commit: Deleted {} nodes", deletes); + trace!("commit: Deleted {} nodes", deletes); Ok(ret as u32) } @@ -304,7 +303,7 @@ impl JournalDB { era -= 1; } } - debug!("Recovered {} counters", res.len()); + trace!("Recovered {} counters", res.len()); res } } From 96617533c8d41ac8726bd17e7de8f3f98b9178d2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 22:57:44 +0100 Subject: [PATCH 19/30] Remove unneeded field. --- ethcore/src/blockchain/blockchain.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e79f1668c..f412a8240 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -40,8 +40,6 @@ pub struct BlockChainConfig { pub pref_cache_size: usize, /// Maximum cache size in bytes. pub max_cache_size: usize, - /// Prefer journal rather than archive. - pub prefer_journal: bool, } impl Default for BlockChainConfig { @@ -49,7 +47,6 @@ impl Default for BlockChainConfig { BlockChainConfig { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, - prefer_journal: false, } } } From 098a6ad2cc3c33810e9036d0a29841a3124a7e18 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:09:05 +0100 Subject: [PATCH 20/30] Reset `HISTORY`. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index bd8c5175b..fb46c81b1 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -207,7 +207,7 @@ pub struct Client where V: Verifier { secret_store: Arc>, } -const HISTORY: u64 = 30; +const HISTORY: u64 = 1000; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { From 86c34c7d10b36f39437289fa439613ae73d3f44a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:29:56 +0100 Subject: [PATCH 21/30] Remove "fix". --- ethcore/src/block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index d05ce51b9..68f647e37 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -220,8 +220,8 @@ impl<'x> OpenBlock<'x> { /// NOTE Will check chain constraints and the uncle number but will NOT check /// that the header itself is actually valid. pub fn push_uncle(&mut self, valid_uncle_header: Header) -> Result<(), BlockError> { - if self.block.base.uncles.len() > self.engine.maximum_uncle_count() { - return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len()})); + if self.block.base.uncles.len() + 1 > self.engine.maximum_uncle_count() { + return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len() + 1})); } // TODO: check number // TODO: check not a direct ancestor (use last_hashes for that) From 8f0005617171f6b7c71c9a2f10fc5e48821248b1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:43:59 +0100 Subject: [PATCH 22/30] Avoid sealing unnecessarily. --- ethcore/src/client.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index fb46c81b1..845702285 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -17,6 +17,7 @@ //! Blockchain database client. use std::marker::PhantomData; +use std::sync::atomic::AtomicBool; use util::*; use util::panics::*; use blockchain::{BlockChain, BlockProvider}; @@ -200,6 +201,7 @@ pub struct Client where V: Verifier { panic_handler: Arc, // for sealing... + sealing_enabled: AtomicBool, sealing_block: Mutex>, author: RwLock
, extra_data: RwLock, @@ -251,6 +253,7 @@ impl Client where V: Verifier { report: RwLock::new(Default::default()), import_lock: Mutex::new(()), panic_handler: panic_handler, + sealing_enabled: AtomicBool::new(false), sealing_block: Mutex::new(None), author: RwLock::new(Address::new()), extra_data: RwLock::new(Vec::new()), @@ -398,7 +401,7 @@ impl Client where V: Verifier { } } - if self.chain_info().best_block_hash != original_best { + if self.chain_info().best_block_hash != original_best && self.sealing_enabled.load(atomic::Ordering::Relaxed) { self.prepare_sealing(); } @@ -481,7 +484,7 @@ impl Client where V: Verifier { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().take(self.engine.deref().deref().maximum_uncle_count()).foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. @@ -493,6 +496,8 @@ impl Client where V: Verifier { /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. pub fn sealing_block(&self) -> &Mutex> { if self.sealing_block.lock().unwrap().is_none() { + self.sealing_enabled.store(true, atomic::Ordering::Relaxed); + // TODO: Above should be on a timer that resets after two blocks have arrived without being asked for. self.prepare_sealing(); } &self.sealing_block From 2d6738fcde45f32d3b5b95e8e6b584f928864f83 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 23:53:57 +0100 Subject: [PATCH 23/30] Additional logging and assert --- util/src/journaldb.rs | 45 ++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index b20934397..e3dd2bbfd 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -122,29 +122,29 @@ impl JournalDB { } /// Drain the overlay and place it into a batch for the DB. - fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> (usize, usize) { - let mut ret = 0usize; + fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize { + let mut inserts = 0usize; let mut deletes = 0usize; for i in overlay.drain().into_iter() { let (key, (value, rc)) = i; if rc > 0 { assert!(rc == 1); batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); - ret += 1; + inserts += 1; } if rc < 0 { assert!(rc == -1); - ret += 1; deletes += 1; } } - (ret, deletes) + trace!("commit: Inserted {}, Deleted {} nodes", inserts, deletes); + inserts + deletes } /// Just commit the overlay into the backing DB. fn commit_without_counters(&mut self) -> Result { let batch = DBTransaction::new(); - let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); + let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); Ok(ret as u32) } @@ -183,14 +183,23 @@ impl JournalDB { let mut index = 0usize; let mut last; - while try!(self.backing.get({ - let mut r = RlpStream::new_list(3); - r.append(&now); - r.append(&index); - r.append(&&PADDING[..]); - last = r.drain(); - &last - })).is_some() { + while { + let record = try!(self.backing.get({ + let mut r = RlpStream::new_list(3); + r.append(&now); + r.append(&index); + r.append(&&PADDING[..]); + last = r.drain(); + &last + })); + match record { + Some(r) => { + assert!(&Rlp::new(&r).val_at::(0) != id); + true + }, + None => false, + } + } { index += 1; } @@ -236,6 +245,7 @@ impl JournalDB { trace!("Purging nodes inserted in non-canon: {:?}", inserts); to_remove.append(&mut inserts); } + trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): {} entries", end_era, index, rlp.val_at::(0), canon_id, to_remove.len()); try!(batch.delete(&last)); index += 1; } @@ -243,18 +253,17 @@ impl JournalDB { let canon_inserts = canon_inserts.drain(..).collect::>(); // Purge removed keys if they are not referenced and not re-inserted in the canon commit let mut deletes = 0; + trace!("Purging filtered notes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); for h in to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)) { try!(batch.delete(&h)); deletes += 1; } - trace!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); + trace!("Total nodes purged: {}", deletes); } // Commit overlay insertions - let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); - + let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); - trace!("commit: Deleted {} nodes", deletes); Ok(ret as u32) } From a4640beb2c766964d5fa905fba0d3eacfb35725c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 5 Mar 2016 00:00:43 +0100 Subject: [PATCH 24/30] Typo --- util/src/journaldb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index e3dd2bbfd..01e53f819 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -253,7 +253,7 @@ impl JournalDB { let canon_inserts = canon_inserts.drain(..).collect::>(); // Purge removed keys if they are not referenced and not re-inserted in the canon commit let mut deletes = 0; - trace!("Purging filtered notes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); + trace!("Purging filtered nodes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); for h in to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)) { try!(batch.delete(&h)); deletes += 1; From 5ad577301420c092454c6268ca3f7a38dbb5ff44 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Mar 2016 10:45:05 +0100 Subject: [PATCH 25/30] verifier improvements --- ethcore/src/client.rs | 4 ++-- ethcore/src/verification/canon_verifier.rs | 6 ++++++ ethcore/src/verification/noop_verifier.rs | 6 ++++++ ethcore/src/verification/verification.rs | 2 +- ethcore/src/verification/verifier.rs | 3 +++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c40ac2ab8..858185873 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -233,7 +233,7 @@ impl Client { impl Client where V: Verifier { /// Create a new client with given spec and DB path and custom verifier. - pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result>, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning @@ -312,7 +312,7 @@ impl Client where V: Verifier { } // Verify Block Family - let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); + let verify_family_result = V::verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); if let Err(e) = verify_family_result { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs index 0d9cbc6b6..30e368f1b 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/src/verification/canon_verifier.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; use super::Verifier; @@ -22,6 +24,10 @@ use super::verification; pub struct CanonVerifier; impl Verifier for CanonVerifier { + fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> { + verification::verify_block_family(header, bytes, engine, bc) + } + fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> { verification::verify_block_final(expected, got) } diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index 8dfd64771..ae2a153fe 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; use super::Verifier; @@ -21,6 +23,10 @@ use super::Verifier; pub struct NoopVerifier; impl Verifier for NoopVerifier { + fn verify_block_family(_header: &Header, _bytes: &[u8], _engine: &Engine, _bc: &BlockProvider) -> Result<(), Error> { + Ok(()) + } + fn verify_block_final(_expected: &Header, _got: &Header) -> Result<(), Error> { Ok(()) } diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 761e2e8cd..ed3db3791 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -78,7 +78,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> } /// Phase 3 verification. Check block information against parent and uncles. -pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BC) -> Result<(), Error> where BC: BlockProvider { +pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> { // TODO: verify timestamp let parent = try!(bc.block_header(&header.parent_hash).ok_or_else(|| Error::from(BlockError::UnknownParent(header.parent_hash.clone())))); try!(verify_parent(&header, &parent)); diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs index 0ffbf3bdd..cc5edce29 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/src/verification/verifier.rs @@ -14,10 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; /// Should be used to verify blocks. pub trait Verifier: Send + Sync { + fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>; fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>; } From 1d04a7b8f98614c4a883585d0b84a35a1a469853 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 13:11:54 +0300 Subject: [PATCH 26/30] changing warning to trace --- util/src/keys/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index bfb8e6c79..625d6fd8f 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -105,7 +105,7 @@ impl SecretStore { import_path.push(".ethereum"); import_path.push("keystore"); if let Err(e) = geth_import::import_geth_keys(self, &import_path) { - warn!(target: "sstore", "Error retrieving geth keys: {:?}", e) + trace!(target: "sstore", "Geth key not imported: {:?}", e); } } From 8a13e87cbeb0905077a25092f088a54482ab2e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 5 Mar 2016 11:30:09 +0100 Subject: [PATCH 27/30] Renaming BlocksWith helper to EachBlockWith --- sync/src/chain.rs | 34 +++++++++++++-------------- sync/src/tests/chain.rs | 48 +++++++++++++++++++-------------------- sync/src/tests/helpers.rs | 10 ++++---- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 983ce62c3..e8ad81a3a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -491,7 +491,7 @@ impl ChainSync { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { - if self.current_base_block() < header.number { + if self.current_base_block() < header.number { self.last_imported_block = Some(header.number); self.remove_downloaded_blocks(header.number); } @@ -1433,7 +1433,7 @@ mod tests { #[test] fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let chain_info = client.chain_info(); @@ -1447,7 +1447,7 @@ mod tests { #[test] fn calculates_tree_for_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(15, BlocksWith::Uncle); + client.add_blocks(15, EachBlockWith::Uncle); let start = client.block_hash_delta_minus(4); let end = client.block_hash_delta_minus(2); @@ -1464,7 +1464,7 @@ mod tests { #[test] fn sends_new_hashes_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1483,7 +1483,7 @@ mod tests { #[test] fn sends_latest_block_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1501,7 +1501,7 @@ mod tests { #[test] fn handles_peer_new_block_mallformed() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let block_data = get_dummy_block(11, client.chain_info().best_block_hash); @@ -1519,7 +1519,7 @@ mod tests { #[test] fn handles_peer_new_block() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); @@ -1537,7 +1537,7 @@ mod tests { #[test] fn handles_peer_new_block_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1553,7 +1553,7 @@ mod tests { #[test] fn handles_peer_new_hashes() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1569,7 +1569,7 @@ mod tests { #[test] fn handles_peer_new_hashes_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1587,7 +1587,7 @@ mod tests { #[test] fn hashes_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1605,7 +1605,7 @@ mod tests { #[test] fn block_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1622,9 +1622,9 @@ mod tests { fn should_add_transactions_to_queue() { // given let mut client = TestBlockChainClient::new(); - client.add_blocks(98, BlocksWith::Uncle); - client.add_blocks(1, BlocksWith::UncleAndTransaction); - client.add_blocks(1, BlocksWith::Transaction); + client.add_blocks(98, EachBlockWith::Uncle); + client.add_blocks(1, EachBlockWith::UncleAndTransaction); + client.add_blocks(1, EachBlockWith::Transaction); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; @@ -1648,7 +1648,7 @@ mod tests { #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1672,7 +1672,7 @@ mod tests { #[test] fn returns_requested_block_headers_reverse() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index b388f508d..58f50916e 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -24,8 +24,8 @@ use super::helpers::*; fn two_peers() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); assert_eq!(net.peer(0).chain.blocks.read().unwrap().deref(), net.peer(1).chain.blocks.read().unwrap().deref()); @@ -35,8 +35,8 @@ fn two_peers() { fn status_after_sync() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync(); let status = net.peer(0).sync.status(); assert_eq!(status.state, SyncState::Idle); @@ -45,8 +45,8 @@ fn status_after_sync() { #[test] fn takes_few_steps() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(100, EachBlockWith::Uncle); let total_steps = net.sync(); assert!(total_steps < 7); } @@ -56,7 +56,7 @@ fn empty_blocks() { ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { - let with = if n % 2 == 0 { BlocksWith::Nothing } else { BlocksWith::Uncle }; + let with = if n % 2 == 0 { EachBlockWith::Nothing } else { EachBlockWith::Uncle }; net.peer_mut(1).chain.add_blocks(5, with.clone()); net.peer_mut(2).chain.add_blocks(5, with); } @@ -69,14 +69,14 @@ fn empty_blocks() { fn forked() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(0).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(0).chain.add_blocks(100, BlocksWith::Nothing); //fork - net.peer_mut(1).chain.add_blocks(200, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(200, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); //fork between 1 and 2 - net.peer_mut(2).chain.add_blocks(10, BlocksWith::Nothing); + net.peer_mut(0).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Nothing); //fork + net.peer_mut(1).chain.add_blocks(200, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(200, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, EachBlockWith::Nothing); // peer 1 has the best chain of 601 blocks let peer1_chain = net.peer(1).chain.numbers.read().unwrap().clone(); net.sync(); @@ -88,8 +88,8 @@ fn forked() { #[test] fn restart() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync_steps(8); @@ -110,8 +110,8 @@ fn status_empty() { #[test] fn status_packet() { let mut net = TestNet::new(2); - net.peer_mut(0).chain.add_blocks(100, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(1, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(1, EachBlockWith::Uncle); net.start(); @@ -124,10 +124,10 @@ fn status_packet() { #[test] fn propagate_hashes() { let mut net = TestNet::new(6); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -150,10 +150,10 @@ fn propagate_hashes() { #[test] fn propagate_blocks() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -165,7 +165,7 @@ fn propagate_blocks() { #[test] fn restart_on_malformed_block() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.peer_mut(1).chain.corrupt_block(6); net.sync_steps(10); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index ca7776bf3..5b53ad90b 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -35,7 +35,7 @@ pub struct TestBlockChainClient { } #[derive(Clone)] -pub enum BlocksWith { +pub enum EachBlockWith { Nothing, Uncle, Transaction, @@ -52,12 +52,12 @@ impl TestBlockChainClient { last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), }; - client.add_blocks(1, BlocksWith::Nothing); // add genesis block + client.add_blocks(1, EachBlockWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } - pub fn add_blocks(&mut self, count: usize, with: BlocksWith) { + pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { let len = self.numbers.read().unwrap().len(); for n in len..(len + count) { let mut header = BlockHeader::new(); @@ -65,7 +65,7 @@ impl TestBlockChainClient { header.parent_hash = self.last_hash.read().unwrap().clone(); header.number = n as BlockNumber; let uncles = match with { - BlocksWith::Uncle | BlocksWith::UncleAndTransaction => { + EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => { let mut uncles = RlpStream::new_list(1); let mut uncle_header = BlockHeader::new(); uncle_header.difficulty = From::from(n); @@ -78,7 +78,7 @@ impl TestBlockChainClient { _ => RlpStream::new_list(0) }; let txs = match with { - BlocksWith::Transaction | BlocksWith::UncleAndTransaction => { + EachBlockWith::Transaction | EachBlockWith::UncleAndTransaction => { let mut txs = RlpStream::new_list(1); let keypair = KeyPair::create().unwrap(); let tx = Transaction { From 9e5f8d44342ccfb93a87c46f82498a2b974c6621 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Mar 2016 11:35:44 +0100 Subject: [PATCH 28/30] build on rust stable --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d2349dae..7213b8f09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: allow_failures: - rust: nightly include: + - rust: stable + env: FEATURES="--features travis-beta" KCOV_FEATURES="" TARGETS="-p ethash -p ethcore-util -p ethcore -p ethsync -p ethcore-rpc -p parity" ARCHIVE_SUFFIX="-${TRAVIS_OS_NAME}-${TRAVIS_TAG}" - rust: beta env: FEATURES="--features travis-beta" KCOV_FEATURES="" TARGETS="-p ethash -p ethcore-util -p ethcore -p ethsync -p ethcore-rpc -p parity" ARCHIVE_SUFFIX="-${TRAVIS_OS_NAME}-${TRAVIS_TAG}" - rust: nightly @@ -52,7 +54,7 @@ after_success: | ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /usr/,/.cargo,/root/.multirust target/kcov target/debug/parity-* && [ $TRAVIS_BRANCH = master ] && [ $TRAVIS_PULL_REQUEST = false ] && - [ $TRAVIS_RUST_VERSION = beta ] && + [ $TRAVIS_RUST_VERSION = stable ] && cargo doc --no-deps --verbose ${KCOV_FEATURES} ${TARGETS} && echo '' > target/doc/index.html && pip install --user ghp-import && From b9a6a70cede04e276aaa9e1ea38ee5cef17d3133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 5 Mar 2016 11:37:19 +0100 Subject: [PATCH 29/30] Renaming bad blocks as retracted --- ethcore/src/client.rs | 2 +- ethcore/src/service.rs | 2 +- sync/src/chain.rs | 11 ++++++----- sync/src/lib.rs | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index ef0356d3e..878bacce9 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -410,7 +410,7 @@ impl Client where V: Verifier { if !good_blocks.is_empty() && block_queue.queue_info().is_empty() { io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { good: good_blocks, - bad: bad_blocks, + retracted: bad_blocks, })).unwrap(); } } diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 756d02407..a80adb0ba 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -30,7 +30,7 @@ pub enum SyncMessage { /// Hashes of blocks imported to blockchain good: Vec, /// Hashes of blocks not imported to blockchain - bad: Vec, + retracted: Vec, }, /// A block is ready BlockVerified, diff --git a/sync/src/chain.rs b/sync/src/chain.rs index e8ad81a3a..ddf30854a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1265,10 +1265,11 @@ impl ChainSync { } /// called when block is imported to chain, updates transactions queue - pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], bad: &[H256]) { + pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], retracted: &[H256]) { fn fetch_transactions(chain: &BlockChainClient, hash: &H256) -> Vec { let block = chain .block(BlockId::Hash(hash.clone())) + // Client should send message after commit to db and inserting to chain. .expect("Expected in-chain blocks."); let block = BlockView::new(&block); block.transactions() @@ -1277,14 +1278,14 @@ impl ChainSync { let chain = io.chain(); let good = good.par_iter().map(|h| fetch_transactions(chain, h)); - let bad = bad.par_iter().map(|h| fetch_transactions(chain, h)); + let retracted = retracted.par_iter().map(|h| fetch_transactions(chain, h)); good.for_each(|txs| { let mut transaction_queue = self.transaction_queue.lock().unwrap(); let hashes = txs.iter().map(|tx| tx.hash()).collect::>(); transaction_queue.remove_all(&hashes, |a| chain.nonce(a)); }); - bad.for_each(|txs| { + retracted.for_each(|txs| { // populate sender for tx in &txs { let _sender = tx.sender(); @@ -1628,7 +1629,7 @@ mod tests { let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; - let bad_blocks = vec![client.block_hash_delta_minus(1)]; + let retracted_blocks = vec![client.block_hash_delta_minus(1)]; let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1637,7 +1638,7 @@ mod tests { sync.chain_new_blocks(&io, &[], &good_blocks); assert_eq!(sync.transaction_queue.lock().unwrap().status().future, 0); assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); - sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); + sync.chain_new_blocks(&io, &good_blocks, &retracted_blocks); // then let status = sync.transaction_queue.lock().unwrap().status(); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 44f3f02e0..d67a09f3b 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -157,9 +157,9 @@ impl NetworkProtocolHandler for EthSync { SyncMessage::BlockVerified => { self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); }, - SyncMessage::NewChainBlocks { ref good, ref bad } => { + SyncMessage::NewChainBlocks { ref good, ref retracted } => { let sync_io = NetSyncIo::new(io, self.chain.deref()); - self.sync.write().unwrap().chain_new_blocks(&sync_io, good, bad); + self.sync.write().unwrap().chain_new_blocks(&sync_io, good, retracted); } } } From cfbaa2d6e99cecbd499fa7fc0eefc510e6338e84 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 14:25:46 +0300 Subject: [PATCH 30/30] fixed namespaces --- util/benches/bigint.rs | 2 +- util/benches/rlp.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index fc41ab628..575164cb6 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -28,7 +28,7 @@ extern crate ethcore_util; extern crate rand; use test::{Bencher, black_box}; -use ethcore_util::uint::*; +use ethcore_util::numbers::*; #[bench] fn u256_add(b: &mut Bencher) { diff --git a/util/benches/rlp.rs b/util/benches/rlp.rs index e94cb3635..4a983f369 100644 --- a/util/benches/rlp.rs +++ b/util/benches/rlp.rs @@ -28,7 +28,7 @@ extern crate ethcore_util; use test::Bencher; use std::str::FromStr; use ethcore_util::rlp::*; -use ethcore_util::uint::U256; +use ethcore_util::numbers::U256; #[bench] fn bench_stream_u64_value(b: &mut Bencher) {