From fcba0122bc5a8116da6c262080ee001bc7ded14b Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 22 Dec 2015 22:19:50 +0100 Subject: [PATCH 01/27] started sync --- Cargo.toml | 2 +- src/bin/client.rs | 17 ++ src/eth.rs | 46 ++++ src/lib.rs | 35 +-- src/sync/chain.rs | 560 ++++++++++++++++++++++++++++++++++++++++++++++ src/sync/mod.rs | 35 +++ 6 files changed, 678 insertions(+), 17 deletions(-) create mode 100644 src/bin/client.rs create mode 100644 src/eth.rs create mode 100644 src/sync/chain.rs create mode 100644 src/sync/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 55e9ca79c..f51fad702 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Ethcore "] [dependencies] log = "0.3" env_logger = "0.3" -ethcore-util = "0.1.0" +ethcore-util = { path = "../ethcore-util" } evmjit = { path = "rust-evmjit", optional = true } rustc-serialize = "0.3" diff --git a/src/bin/client.rs b/src/bin/client.rs new file mode 100644 index 000000000..5bc2adaab --- /dev/null +++ b/src/bin/client.rs @@ -0,0 +1,17 @@ +extern crate ethcore_util as util; + +use std::io::*; +use util::network::{NetworkService}; + + +fn main() { + let mut service = NetworkService::start().unwrap(); + loop { + let mut cmd = String::new(); + stdin().read_line(&mut cmd).unwrap(); + if cmd == "quit" || cmd == "exit" || cmd == "q" { + break; + } + } +} + diff --git a/src/eth.rs b/src/eth.rs new file mode 100644 index 000000000..772a59277 --- /dev/null +++ b/src/eth.rs @@ -0,0 +1,46 @@ +use util::hash::H256; +use util::bytes::Bytes; +use util::uint::U256; + +pub enum QueueStatus { + /// Part of the known chain + Known, + /// Part of the unknown chain + Unknown, +} + +pub enum BlockStatus { + InChain, + Queued(QueueStatus), + Bad, +} + +pub struct BlockChainInfo { + pub total_difficulty: U256, + pub pending_total_difficulty: U256, + pub genesis_hash: H256, + pub last_block_hash: H256, + pub last_block_number: BlockNumber +} + +pub struct BlockQueueStats; +pub struct TreeRoute; + +pub type BlockNumber = u32; +pub type BlockHeader = ::blockheader::Header; + +pub trait BlockChainClient : Sync { + fn block_header(&self, h: &H256) -> Option; + fn block_body(&self, h: &H256) -> Option; + fn block(&self, h: &H256) -> Option; + fn block_status(&self, h: &H256) -> BlockStatus; + fn block_header_at(&self, n: BlockNumber) -> Option; + fn block_body_at(&self, h: BlockNumber) -> Option; + fn block_at(&self, h: BlockNumber) -> Option; + fn block_status_at(&self, h: BlockNumber) -> Option; + fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; + fn import_block(&mut self, b: Bytes) -> BlockStatus; + fn queue_stats(&self) -> BlockQueueStats; + fn clear_queue(&mut self) -> BlockQueueStats; + fn info(&self) -> BlockChainInfo; +} diff --git a/src/lib.rs b/src/lib.rs index d7f354423..a95cd931b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ //! Ethcore's ethereum implementation -//! +//! //! ### Rust version //! - beta //! - nightly @@ -7,44 +7,44 @@ //! ### Supported platforms: //! - OSX //! - Linux/Ubuntu -//! +//! //! ### Dependencies: //! - RocksDB 3.13 //! - LLVM 3.7 (optional, required for `jit`) //! - evmjit (optional, required for `jit`) //! //! ### Dependencies Installation -//! +//! //! - OSX -//! +//! //! - rocksdb //! ```bash //! brew install rocksdb //! ``` -//! +//! //! - llvm -//! +//! //! - download llvm 3.7 from http://llvm.org/apt/ //! //! ```bash //! cd llvm-3.7.0.src //! mkdir build && cd $_ -//! cmake -G "Unix Makefiles" .. -DCMAKE_C_FLAGS_RELEASE= -DCMAKE_CXX_FLAGS_RELEASE= -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/llvm/3.7 -DCMAKE_BUILD_TYPE=Release +//! cmake -G "Unix Makefiles" .. -DCMAKE_C_FLAGS_RELEASE= -DCMAKE_CXX_FLAGS_RELEASE= -DCMAKE_INSTALL_PREFIX=/usr/local/Cellar/llvm/3.7 -DCMAKE_BUILD_TYPE=Release //! make && make install //! ``` //! - evmjit -//! +//! //! - download from https://github.com/debris/evmjit -//! +//! //! ```bash //! cd evmjit //! mkdir build && cd $_ //! cmake -DLLVM_DIR=/usr/local/lib/llvm-3.7/share/llvm/cmake .. //! make && make install //! ``` -//! +//! //! - Linux/Ubuntu -//! +//! //! - rocksdb //! //! ```bash @@ -52,15 +52,15 @@ //! tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib //! sudo make install //! ``` -//! +//! //! - llvm -//! +//! //! - install using packages from http://llvm.org/apt/ -//! +//! //! - evmjit -//! +//! //! - download from https://github.com/debris/evmjit -//! +//! //! ```bash //! cd evmjit //! mkdir build && cd $_ @@ -88,6 +88,9 @@ pub mod blockheader; pub mod transaction; pub mod networkparams; pub mod denominations; +pub mod eth; + +pub mod sync; #[test] fn it_works() { diff --git a/src/sync/chain.rs b/src/sync/chain.rs new file mode 100644 index 000000000..51b210ae9 --- /dev/null +++ b/src/sync/chain.rs @@ -0,0 +1,560 @@ +use std::collections::{HashSet, HashMap}; +use std::cmp::{min, max}; +use std::ops::{Add, Sub}; +use std::mem::{replace}; +use util::network::{PeerId, HandlerIo, PacketId}; +use util::hash::{H256}; +use util::bytes::{Bytes}; +use util::uint::{U256}; +use util::rlp::{Rlp, RlpStream}; +use util::rlp::rlptraits::{Stream, View}; +use eth::{BlockNumber, BlockChainClient}; + +pub struct SyncIo<'s, 'h> where 'h:'s { + network: &'s mut HandlerIo<'h>, + chain: &'s mut BlockChainClient +} + +impl<'s, 'h> SyncIo<'s, 'h> { + fn disable_peer(&mut self, peer_id: &PeerId) { + self.network.disable_peer(*peer_id); + } +} + +const STATUS_PACKET: u8 = 0x00; +const NEW_BLOCK_HASHES_PACKET: u8 = 0x01; +const TRANSACTIONS_PACKET: u8 = 0x02; +const GET_BLOCK_HEADERS_PACKET: u8 = 0x03; +const BLOCK_HEADERS_PACKET: u8 = 0x04; +const GET_BLOCK_BODIES_PACKET: u8 = 0x05; +const BLOCK_BODIES_PACKET: u8 = 0x06; +const NEW_BLOCK_PACKET: u8 = 0x07; + +const GET_NODE_DATA_PACKET: u8 = 0x0d; +const NODE_DATA_PACKET: u8 = 0x0e; +const GET_RECEIPTS_PACKET: u8 = 0x0f; +const RECEIPTS_PACKET: u8 = 0x10; + +struct Header { + ///Header data + data: Bytes, + /// Block hash + hash: H256, + /// Parent hash + parent: H256, +} + +/// Used to identify header by transactions and uncles hashes +#[derive(Eq, PartialEq, Hash)] +struct HeaderId { + transactions_root: H256, + uncles: H256 +} + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum SyncState { + /// Initial chain sync has not started yet + NotSynced, + /// Initial chain sync complete. Waiting for new packets + Idle, + /// Block downloading paused. Waiting for block queue to process blocks and free space + Waiting, + /// Downloading blocks + Blocks, + /// Downloading blocks learned from NewHashes packet + NewBlocks, +} + +pub struct SyncStatus { + state: SyncState, + protocol_version: u8, + start_block_number: BlockNumber, + current_block_number: BlockNumber, + highest_block_number: BlockNumber, + blocks_total: usize, + blocks_received: usize +} + +#[derive(PartialEq, Eq, Debug)] +enum PeerAsking +{ + Nothing, + State, + BlockHeaders, + BlockBodies, +} + +struct PeerInfo { + protocol_version: u32, + genesis: H256, + network_id: U256, + latest: H256, + difficulty: U256, + asking: PeerAsking, + asking_blocks: Vec +} + +type Body = Bytes; + +pub struct ChainSync { + /// Sync state + state: SyncState, + /// Last block number for the start of sync + starting_block: BlockNumber, + /// Highest block number seen + highest_block: BlockNumber, + /// Set of block header numbers being downloaded + downloading_headers: HashSet, + /// Set of block body numbers being downloaded + downloading_bodies: HashSet, + /// Downloaded headers. + headers: Vec<(BlockNumber, Vec
)>, //TODO: use BTreeMap once range API is sable. For now it a vector sorted in descending order + /// Downloaded bodies + bodies: Vec<(BlockNumber, Vec)>, //TODO: use BTreeMap once range API is sable. For now it a vector sorted in descending order + /// Peer info + peers: HashMap, + /// Used to map body to header + header_ids: HashMap, + /// Last impoted block number + last_imported_block: BlockNumber, + /// Syncing total difficulty + syncing_difficulty: U256, + /// True if common block for our and remote chain has been found + have_common_block: bool, +} + + +impl ChainSync { + + pub fn new(io: &mut SyncIo) -> ChainSync { + let mut sync = ChainSync { + state: SyncState::NotSynced, + starting_block: 0, + highest_block: 0, + downloading_headers: HashSet::new(), + downloading_bodies: HashSet::new(), + headers: Vec::new(), + bodies: Vec::new(), + peers: HashMap::new(), + header_ids: HashMap::new(), + last_imported_block: 0, + syncing_difficulty: U256::from(0u64), + have_common_block: false + }; + sync.restart(io); + sync + } + + /// @returns Synchonization status + pub fn status(&self) -> SyncStatus { + SyncStatus { + state: self.state.clone(), + protocol_version: 63, + start_block_number: self.starting_block, + current_block_number: 0, //TODO: + highest_block_number: self.highest_block, + blocks_total: (self.last_imported_block - self.starting_block) as usize, + blocks_received: (self.highest_block - self.starting_block) as usize + } + } + + /// Abort all sync activity + pub fn abort(&mut self, io: &mut SyncIo) { + self.restart(io); + self.peers.clear(); + } + + /// @returns true is Sync is in progress + pub fn is_syncing(&self) { + self.state != SyncState::Idle; + } + + fn reset(&mut self) { + self.downloading_headers.clear(); + self.downloading_bodies.clear(); + self.headers.clear(); + self.bodies.clear(); + for (_, ref mut p) in self.peers.iter_mut() { + p.asking_blocks.clear(); + } + self.header_ids.clear(); + self.syncing_difficulty = From::from(0u64); + self.state = SyncState::Idle; + } + + /// Restart sync + pub fn restart(&mut self, io: &mut SyncIo) { + self.reset(); + self.last_imported_block = 0; + self.starting_block = 0; + self.highest_block = 0; + self.have_common_block = false; + io.chain.clear_queue(); + self.starting_block = io.chain.info().last_block_number; + self.state = SyncState::NotSynced; + } + + /// Called by peer to report status + pub fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let peer = PeerInfo { + protocol_version: r.val_at(0), + network_id: r.val_at(1), + difficulty: r.val_at(2), + latest: r.val_at(3), + genesis: r.val_at(4), + asking: PeerAsking::Nothing, + asking_blocks: Vec::new(), + }; + let old = self.peers.insert(peer_id.clone(), peer); + if old.is_some() { + panic!("ChainSync: new peer already exists"); + } + self.sync_peer(io, peer_id, false); + } + + /// Called by peer once it has new block headers during sync + pub fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let item_count = r.item_count(); + trace!(target: "sync", "BlockHeaders ({} entries)", item_count); + self.clear_peer_download(peer_id); + if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { + trace!(target: "sync", "Ignored unexpected block headers"); + return; + } + if self.state == SyncState::Waiting { + trace!(target: "sync", "Ignored block headers while waiting"); + return; + } + } + + /// Called by peer once it has new block bodies + pub fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + } + + /// Called by peer once it has new block bodies + pub fn on_peer_new_block(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + } + + pub fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + } + + /// Called by peer when it is disconnecting + pub fn on_peer_aborting(&mut self, peer: &PeerId) { + } + + /// Resume downloading after witing state + fn continue_sync(&mut self) { + } + + /// Called after all blocks have been donloaded + fn complete_sync(&mut self) { + } + + /// Enter waiting state + fn pause_sync(&mut self) { + } + + fn reset_sync(&mut self) { + } + + fn sync_peer(&mut self, io: &mut SyncIo, peer_id: &PeerId, force: bool) { + let (peer_latest, peer_difficulty) = { + let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + if peer.asking != PeerAsking::Nothing + { + debug!(target: "sync", "Can't sync with this peer - outstanding asks."); + return; + } + if self.state == SyncState::Waiting + { + debug!(target: "sync", "Waiting for block queue"); + return; + } + (peer.latest.clone(), peer.difficulty.clone()) + }; + + let td = io.chain.info().pending_total_difficulty; + let syncing_difficulty = max(self.syncing_difficulty, td); + if force || peer_difficulty > syncing_difficulty { + // start sync + self.syncing_difficulty = peer_difficulty; + if self.state == SyncState::Idle || self.state == SyncState::NotSynced { + self.state = SyncState::Blocks; + } + self.request_headers_by_hash(io, peer_id, &peer_latest, 1, 0, false); + } + else if self.state == SyncState::Blocks { + self.request_blocks(io, peer_id); + } + } + + fn request_blocks(&mut self, io: &mut SyncIo, peer_id: &PeerId) { + self.clear_peer_download(peer_id); + // check to see if we need to download any block bodies first + let mut needed_bodies: Vec = Vec::new(); + let mut needed_numbers: Vec = Vec::new(); + let mut index = 0usize; + + if self.have_common_block && !self.headers.is_empty() && self.headers.last().unwrap().0 == self.last_imported_block + 1 { + let mut header = self.headers.len() - 1; + while header != 0 && needed_bodies.len() < 1024 && index < self.headers[header].1.len() { + let block = self.headers[header].0 + index as BlockNumber; + if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { + needed_bodies.push(self.headers[header].1[index].hash.clone()); + needed_numbers.push(block); + self.downloading_bodies.insert(block); + } + index += 1; + if index >= self.headers[header].1.len() { + index = 0; + header -= 1; + } + } + } + if !needed_bodies.is_empty() { + replace(&mut self.peers.get_mut(peer_id).unwrap().asking_blocks, needed_numbers); + self.request_bodies(io, peer_id, needed_bodies); + } + else { + // check if need to download headers + let mut start = 0usize; + if !self.have_common_block { + // download backwards until common block is found 1 header at a time + start = io.chain.info().last_block_number as usize; + if !self.headers.is_empty() { + start = min(start, self.headers.last().unwrap().0 as usize - 1); + } + if start <= 1 { + self.have_common_block = true; //reached genesis + } + } + if self.have_common_block { + start = self.last_imported_block as usize + 1; + let mut next = self.headers.len() - 1; + let mut count = 0usize; + if !self.headers.is_empty() && start >= self.headers.last().unwrap().0 as usize { + start = self.headers.last().unwrap().0 as usize + self.headers.last().unwrap().1.len(); + next -=1; + } + while count == 0 && next != 0 { + count = min(1024, self.headers[next].0 as usize - start); + while count > 0 && self.downloading_headers.contains(&(start as BlockNumber)) { + start +=1; + count -=1; + } + } + let mut headers: Vec = Vec::new(); + for block in start..(start + count) { + if !self.downloading_headers.contains(&(block as BlockNumber)) { + headers.push(block as BlockNumber); + self.downloading_headers.insert(block as BlockNumber); + } + } + count = self.headers.len(); + if count > 0 { + replace(&mut self.peers.get_mut(peer_id).unwrap().asking_blocks, headers); + assert!(!self.headers.have_item(&(start as BlockNumber))); + self.request_headers_by_number(io, peer_id, start as BlockNumber, count, 0, false); + } + else if start >= (self.headers[next].0 as usize) { + start = self.headers[next].0 as usize + self.headers[next].1.len(); + next -=1; + } + } + else { + self.request_headers_by_number(io, peer_id, start as BlockNumber, 1, 0, false); + } + } + } + + fn clear_peer_download(&mut self, peer: &PeerId) { + } + fn clear_all_download(&mut self) { + } + fn collect_blocks(&mut self) { + } + + fn request_headers_by_hash(&mut self, sync: &mut SyncIo, peer_id: &PeerId, h: &H256, count: usize, skip: usize, reverse: bool) { + let mut rlp = RlpStream::new_list(4); + rlp.append(h); + rlp.append(&count); + rlp.append(&skip); + rlp.append(&if reverse {1u32} else {0u32}); + self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); + } + + fn request_headers_by_number(&mut self, sync: &mut SyncIo, peer_id: &PeerId, n: BlockNumber, count: usize, skip: usize, reverse: bool) { + let mut rlp = RlpStream::new_list(4); + rlp.append(&n); + rlp.append(&count); + rlp.append(&skip); + rlp.append(&if reverse {1u32} else {0u32}); + self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); + } + + fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: &PeerId, hashes: Vec) { + let mut rlp = RlpStream::new_list(hashes.len()); + for h in hashes { + rlp.append(&h); + } + self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_BODIES_PACKET, rlp.out()); + } + + fn send_request(&mut self, sync: &mut SyncIo, peer_id: &PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { + { + let mut peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + if peer.asking != PeerAsking::Nothing { + warn!(target:"sync", "Asking {:?} while requesting {:?}", asking, peer.asking); + } + } + match sync.network.send(*peer_id, packet_id, packet) { + Err(e) => { + warn!(target:"sync", "Error sending request: {:?}", e); + sync.disable_peer(peer_id); + self.on_peer_aborting(peer_id); + } + Ok(_) => { + let mut peer = self.peers.get_mut(&peer_id).unwrap(); + peer.asking = asking; + } + } + } +} + +pub trait ToUsize { + fn to_usize(&self) -> usize; +} + +pub trait FromUsize { + fn from(s: usize) -> Self; +} + +impl ToUsize for BlockNumber { + fn to_usize(&self) -> usize { + *self as usize + } +} + +impl FromUsize for BlockNumber { + fn from(s: usize) -> BlockNumber { + s as BlockNumber + } +} + +pub trait RangeCollection { + fn have_item(&self, key: &K) -> bool; + fn find_item(&self, key: &K) -> Option<&V>; + fn remove_head(&mut self, start: &K); + fn remove_tail(&mut self, start: &K); + fn insert_item(&mut self, key: K, value: V); +} + +impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Add + Sub + Copy + FromUsize + ToUsize { + fn have_item(&self, key: &K) -> bool { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(_) => true, + Err(index) => match self.get(index + 1) { + Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from(v.len())) > *key, + _ => false + }, + } + } + + fn find_item(&self, key: &K) -> Option<&V> { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => self.get(index).unwrap().1.get(0), + Err(index) => match self.get(index + 1) { + Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from(v.len())) > *key => v.get((*key - *k).to_usize()), + _ => None + }, + } + } + + /// Remove element key and following elements in the same range + fn remove_tail(&mut self, key: &K) { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => { self.remove(index); }, + Err(index) =>{ + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from(v.len())) > *key => { + v.truncate((*key - *k).to_usize()); + empty = v.is_empty(); + } + _ => {} + } + if empty { + self.remove(index + 1); + } + }, + } + } + + /// Remove range elements up to key + fn remove_head(&mut self, key: &K) { + if *key == FromUsize::from(0) { + return + } + + let prev = *key - FromUsize::from(1); + match self.binary_search_by(|&(k, _)| k.cmp(&prev).reverse()) { + Ok(index) => { self.remove(index); }, + Err(index) => { + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from(v.len())) > *key => { + let head = v.split_off((*key - *k).to_usize()); + empty = head.is_empty(); + let removed = ::std::mem::replace(v, head); + let new_k = *k - FromUsize::from(removed.len()); + ::std::mem::replace(k, new_k); + } + _ => {} + } + if empty { + self.remove(index + 1); + } + }, + } + } + + fn insert_item(&mut self, key: K, value: V) { + assert!(!self.have_item(&key)); + + let mut lower = match self.binary_search_by(|&(k, _)| k.cmp(&key).reverse()) { + Ok(index) => index, + Err(index) => index, + }; + + lower += 1; + + let mut to_remove: Option = None; + if lower < self.len() && self[lower].0 + FromUsize::from(self[lower].1.len()) == key { + // extend into existing chunk + self[lower].1.push(value); + } + else { + // insert a new chunk + let mut range: Vec = vec![value]; + self.insert(lower, (key, range)); + }; + let next = lower - 1; + if next < self.len() + { + { + let (mut next, mut inserted) = self.split_at_mut(lower); + let mut next = next.last_mut().unwrap(); + let mut inserted = inserted.first_mut().unwrap(); + if next.0 == key + FromUsize::from(1) + { + inserted.1.append(&mut next.1); + to_remove = Some(lower - 1); + } + } + + if let Some(r) = to_remove { + self.remove(r); + } + } + } +} diff --git a/src/sync/mod.rs b/src/sync/mod.rs new file mode 100644 index 000000000..29e61b7d0 --- /dev/null +++ b/src/sync/mod.rs @@ -0,0 +1,35 @@ +use std::sync::{Arc}; +use eth::{BlockChainClient}; +use util::network::{Error as NetworkError, ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId}; + +mod chain; + + +pub fn new(service: &mut NetworkService, eth_cleint: Arc) { + +} + +struct EthSync { + idle: bool, + chain: Arc +} + +impl ProtocolHandler for EthSync { + fn initialize(&mut self, io: &mut HandlerIo) { + io.register_timer(1000); + } + + fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { + } + + fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { + } + + fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { + } + + fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { + } +} + + From 0578712a26356591e6ccb81c1675363384da03fb Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 24 Dec 2015 17:18:47 +0100 Subject: [PATCH 02/27] syncing code done --- src/eth.rs | 18 +- src/header.rs | 6 +- src/sync/chain.rs | 464 +++++++++++++++++++++++++++++++++++++++++++--- src/sync/mod.rs | 9 +- 4 files changed, 461 insertions(+), 36 deletions(-) diff --git a/src/eth.rs b/src/eth.rs index 7b3de0d9d..3a8853bac 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -13,6 +13,14 @@ pub enum BlockStatus { InChain, Queued(QueueStatus), Bad, + Unknown, +} + +pub enum ImportResult { + Queued(QueueStatus), + AlreadyInChain, + AlreadyQueued(QueueStatus), + Bad, } pub struct BlockChainInfo { @@ -30,16 +38,18 @@ pub type BlockNumber = u32; pub type BlockHeader = ::header::Header; pub trait BlockChainClient : Sync { - fn block_header(&self, h: &H256) -> Option; + fn block_header(&self, h: &H256) -> Option; fn block_body(&self, h: &H256) -> Option; fn block(&self, h: &H256) -> Option; fn block_status(&self, h: &H256) -> BlockStatus; - fn block_header_at(&self, n: BlockNumber) -> Option; + fn block_header_at(&self, n: BlockNumber) -> Option; fn block_body_at(&self, h: BlockNumber) -> Option; fn block_at(&self, h: BlockNumber) -> Option; - fn block_status_at(&self, h: BlockNumber) -> Option; + fn block_status_at(&self, h: BlockNumber) -> BlockStatus; fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; - fn import_block(&mut self, b: Bytes) -> BlockStatus; + fn state_data(&self, h: &H256) -> Option; + fn block_receipts(&self, h: &H256) -> Option; + fn import_block(&mut self, b: &[u8]) -> ImportResult; fn queue_stats(&self) -> BlockQueueStats; fn clear_queue(&mut self) -> BlockQueueStats; fn info(&self) -> BlockChainInfo; diff --git a/src/header.rs b/src/header.rs index 28e568e8e..e161ac280 100644 --- a/src/header.rs +++ b/src/header.rs @@ -52,6 +52,10 @@ impl Header { seal: vec![], } } + + pub fn hash(&self) -> H256 { + unimplemented!(); + } } impl Decodable for Header { @@ -99,7 +103,7 @@ impl Encodable for Header { self.gas_used.encode(e); self.timestamp.encode(e); self.extra_data.encode(e); - + for b in self.seal.iter() { b.encode(e); } diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 51b210ae9..c14abb89a 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -1,14 +1,15 @@ use std::collections::{HashSet, HashMap}; use std::cmp::{min, max}; -use std::ops::{Add, Sub}; +use std::ops::{Add, Sub, Range}; use std::mem::{replace}; use util::network::{PeerId, HandlerIo, PacketId}; use util::hash::{H256}; use util::bytes::{Bytes}; use util::uint::{U256}; -use util::rlp::{Rlp, RlpStream}; +use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp use util::rlp::rlptraits::{Stream, View}; -use eth::{BlockNumber, BlockChainClient}; +use util::sha3::Hashable; +use eth::{BlockNumber, BlockChainClient, BlockHeader, BlockStatus, QueueStatus, ImportResult}; pub struct SyncIo<'s, 'h> where 'h:'s { network: &'s mut HandlerIo<'h>, @@ -16,11 +17,23 @@ pub struct SyncIo<'s, 'h> where 'h:'s { } impl<'s, 'h> SyncIo<'s, 'h> { + pub fn new(network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient) -> SyncIo<'s,'h> { + SyncIo { + network: network, + chain: chain, + } + } fn disable_peer(&mut self, peer_id: &PeerId) { self.network.disable_peer(*peer_id); } } +const PROTOCOL_VERSION: u8 = 63u8; +const MAX_BODIES_TO_SEND: usize = 256; +const MAX_HEADERS_TO_SEND: usize = 1024; +const MAX_NODE_DATA_TO_SEND: usize = 1024; +const MAX_RECEIPTS_TO_SEND: usize = 1024; + const STATUS_PACKET: u8 = 0x00; const NEW_BLOCK_HASHES_PACKET: u8 = 0x01; const TRANSACTIONS_PACKET: u8 = 0x02; @@ -125,7 +138,6 @@ pub struct ChainSync { impl ChainSync { - pub fn new(io: &mut SyncIo) -> ChainSync { let mut sync = ChainSync { state: SyncState::NotSynced, @@ -195,7 +207,7 @@ impl ChainSync { } /// Called by peer to report status - pub fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { let peer = PeerInfo { protocol_version: r.val_at(0), network_id: r.val_at(1), @@ -213,7 +225,7 @@ impl ChainSync { } /// Called by peer once it has new block headers during sync - pub fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { let item_count = r.item_count(); trace!(target: "sync", "BlockHeaders ({} entries)", item_count); self.clear_peer_download(peer_id); @@ -225,36 +237,193 @@ impl ChainSync { trace!(target: "sync", "Ignored block headers while waiting"); return; } + + for i in 0..item_count { + let info: BlockHeader = r.val_at(i); + let number = BlockNumber::from(info.number); + if number <= self.last_imported_block || self.headers.have_item(&number) { + trace!(target: "sync", "Skipping existing block header"); + continue; + } + if number > self.highest_block { + self.highest_block = number; + } + match io.chain.block_status(&info.hash()) { + BlockStatus::InChain => { + self.have_common_block = true; + self.last_imported_block = number; + }, + _ => { + if self.have_common_block { + //validate chain + if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash) { + // mismatching parent id, delete the previous block and don't add this one + // TODO: lower peer rating + debug!(target: "sync", "Mismatched block header {} {}", number, info.hash()); + self.remove_downloaded_blocks(number - 1); + continue; + } + if self.headers.find_item(&(number + 1)).map_or(false, |p| p.parent != info.hash()) { + // mismatching parent id for the next block, clear following headers + debug!(target: "sync", "Mismatched block header {}", number + 1); + self.remove_downloaded_blocks(number + 1); + } + } + let hdr = Header { + data: r.at(i).data().to_vec(), + hash: info.hash(), + parent: info.parent_hash, + }; + self.headers.insert_item(number, hdr); + let header_id = HeaderId { + transactions_root: info.transactions_root, + uncles: info.uncles_hash + }; + if header_id.transactions_root == rlp::SHA3_NULL_RLP && header_id.uncles == rlp::SHA3_EMPTY_LIST_RLP { + //empty body, just mark as downloaded + let mut body_stream = RlpStream::new_list(2); + body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); + body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); + self.bodies.insert_item(number, body_stream.out()); + } + else { + self.header_ids.insert(header_id, number); + } + } + } + self.collect_blocks(io); + self.continue_sync(io); + } } /// Called by peer once it has new block bodies - pub fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let item_count = r.item_count(); + trace!(target: "sync", "BlockBodies ({} entries)", item_count); + self.clear_peer_download(peer_id); + if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { + trace!(target: "sync", "Ignored unexpected block bodies"); + return; + } + if self.state == SyncState::Waiting { + trace!(target: "sync", "Ignored block bodies while waiting"); + return; + } + for i in 0..item_count { + let body: Rlp = r.at(i); + let tx = body.at(0); + let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.data().to_vec()).collect()); //TODO: get rid of vectors here + let uncles = body.at(1).data().sha3(); + let header_id = HeaderId { + transactions_root: tx_root, + uncles: uncles + }; + match self.header_ids.get(&header_id).map(|n| *n) { + Some(n) => { + self.header_ids.remove(&header_id); + self.bodies.insert_item(n, body.data().to_vec()); + } + None => { + debug!(target: "sync", "Ignored unknown block body"); + } + } + } + self.collect_blocks(io); + self.continue_sync(io); } /// Called by peer once it has new block bodies - pub fn on_peer_new_block(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let block_rlp = r.at(0); + let header_rlp = block_rlp.at(0); + let h = header_rlp.data().sha3(); + + match io.chain.import_block(block_rlp.data()) { + ImportResult::AlreadyInChain => { + trace!(target: "sync", "New block already in chain {:?}", h); + }, + ImportResult::AlreadyQueued(_) => { + trace!(target: "sync", "New block already queued {:?}", h); + }, + ImportResult::Queued(QueueStatus::Known) => { + trace!(target: "sync", "New block queued {:?}", h); + }, + ImportResult::Queued(QueueStatus::Unknown) => { + trace!(target: "sync", "New block unknown {:?}", h); + //TODO: handle too many unknown blocks + let difficulty: U256 = r.val_at(1); + let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; + if difficulty > peer_difficulty { + trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); + self.sync_peer(io, peer_id, true); + } + }, + ImportResult::Bad =>{ + debug!(target: "sync", "Bad new block {:?}", h); + io.disable_peer(peer_id); + } + } } - pub fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer: &PeerId, r: &Rlp) { + fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + if self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").asking != PeerAsking::Nothing { + trace!(target: "sync", "Ignoring new hashes since we're already downloading."); + return; + } + let hashes = r.iter().map(|item| (item.val_at::(0), item.val_at::(1))); + let mut max_height: U256 = From::from(0); + for (h, d) in hashes { + match io.chain.block_status(&h) { + BlockStatus::InChain => { + trace!(target: "sync", "New block hash already in chain {:?}", h); + }, + BlockStatus::Queued(_) => { + trace!(target: "sync", "New hash block already queued {:?}", h); + }, + BlockStatus::Unknown => { + trace!(target: "sync", "New unknown block hash {:?}", h); + if d > max_height { + let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + peer.latest = h.clone(); + max_height = d; + } + }, + BlockStatus::Bad =>{ + debug!(target: "sync", "Bad new block hash {:?}", h); + io.disable_peer(peer_id); + return; + } + } + } } /// Called by peer when it is disconnecting - pub fn on_peer_aborting(&mut self, peer: &PeerId) { + pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: &PeerId) { + self.clear_peer_download(peer); + self.continue_sync(io); + } + + pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: &PeerId) { + self.send_status(io, peer); } /// Resume downloading after witing state - fn continue_sync(&mut self) { + fn continue_sync(&mut self, io: &mut SyncIo) { + let peers: Vec = self.peers.keys().map(|k| *k).collect(); + for p in peers { + self.sync_peer(io, &p, false); + } } /// Called after all blocks have been donloaded fn complete_sync(&mut self) { + self.reset(); + self.state = SyncState::Idle; } /// Enter waiting state fn pause_sync(&mut self) { - } - - fn reset_sync(&mut self) { + self.state = SyncState::Waiting; } fn sync_peer(&mut self, io: &mut SyncIo, peer_id: &PeerId, force: bool) { @@ -367,11 +536,90 @@ impl ChainSync { } } - fn clear_peer_download(&mut self, peer: &PeerId) { + fn clear_peer_download(&mut self, peer_id: &PeerId) { + let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + for b in &peer.asking_blocks { + self.downloading_headers.remove(&b); + self.downloading_bodies.remove(&b); + } + peer.asking_blocks.clear(); } - fn clear_all_download(&mut self) { + + fn collect_blocks(&mut self, io: &mut SyncIo) { + if !self.have_common_block || self.headers.is_empty() || self.bodies.is_empty() { + return; + } + + let mut restart = false; + // merge headers and bodies + { + let mut headers = self.headers.last().unwrap(); + let mut bodies = self.bodies.last().unwrap(); + if headers.0 != bodies.0 || headers.0 != self.last_imported_block + 1 { + return; + } + + for i in 0..min(headers.1.len(), bodies.1.len()) { + let mut block_rlp = RlpStream::new_list(3); + block_rlp.append_raw(&headers.1[i].data, 1); + block_rlp.append_raw(&bodies.1[i], 2); + let h = &headers.1[i].hash; + match io.chain.import_block(&block_rlp.out()) { + ImportResult::AlreadyInChain => { + trace!(target: "sync", "Block already in chain {:?}", h); + self.last_imported_block = headers.0 + i as BlockNumber; + }, + ImportResult::AlreadyQueued(_) => { + trace!(target: "sync", "Block already queued {:?}", h); + self.last_imported_block = headers.0 + i as BlockNumber; + }, + ImportResult::Queued(QueueStatus::Known) => { + trace!(target: "sync", "Block queued {:?}", h); + self.last_imported_block = headers.0 + i as BlockNumber; + }, + ImportResult::Queued(QueueStatus::Unknown) => { + panic!("Queued out of order block"); + }, + ImportResult::Bad =>{ + debug!(target: "sync", "Bad block {:?}", h); + restart = true; + } + } + } + } + + if restart { + self.restart(io); + return; + } + + self.headers.remove_head(&self.last_imported_block); + self.bodies.remove_head(&self.last_imported_block); + + if self.headers.is_empty() { + assert!(self.bodies.is_empty()); + self.complete_sync(); + } } - fn collect_blocks(&mut self) { + + fn remove_downloaded_blocks(&mut self, start: BlockNumber) { + for n in self.headers.get_tail(&start) { + match self.headers.find_item(&n) { + Some(ref header_data) => { + let header_to_delete: BlockHeader = rlp::decode(&header_data.data); + let header_id = HeaderId { + transactions_root: header_to_delete.transactions_root, + uncles: header_to_delete.uncles_hash + }; + self.header_ids.remove(&header_id); + }, + None => {} + } + self.downloading_bodies.remove(&n); + self.downloading_headers.remove(&n); + } + self.headers.remove_tail(&start); + self.bodies.remove_tail(&start); } fn request_headers_by_hash(&mut self, sync: &mut SyncIo, peer_id: &PeerId, h: &H256, count: usize, skip: usize, reverse: bool) { @@ -411,7 +659,7 @@ impl ChainSync { Err(e) => { warn!(target:"sync", "Error sending request: {:?}", e); sync.disable_peer(peer_id); - self.on_peer_aborting(peer_id); + self.on_peer_aborting(sync, peer_id); } Ok(_) => { let mut peer = self.peers.get_mut(&peer_id).unwrap(); @@ -419,6 +667,147 @@ impl ChainSync { } } } + + fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + } + + fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { + let mut packet = RlpStream::new_list(5); + let chain = io.chain.info(); + packet.append(&(PROTOCOL_VERSION as u32)); + packet.append(&0u32); //TODO: network id + packet.append(&chain.total_difficulty); + packet.append(&chain.last_block_hash); + packet.append(&chain.genesis_hash); + self.send_request(io, peer_id, PeerAsking::State, STATUS_PACKET, packet.out()); + } + + fn return_block_headers(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + // Packet layout: + // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] + let max_headers: usize = r.val_at(1); + let skip: usize = r.val_at(2); + let reverse: bool = r.val_at(3); + let mut packet = RlpStream::new(); + let last = io.chain.info().last_block_number; + let mut number = if r.at(0).size() == 32 { + // id is a hash + let hash: H256 = r.val_at(0); + trace!(target: "sync", "GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); + match io.chain.block_header(&hash) { + Some(hdr) => From::from(rlp::decode::(&hdr).number), + None => last + } + } + else { + r.val_at(0) + }; + + number = max(1, number); + number = min(last, number); + let max_count = min(MAX_HEADERS_TO_SEND, max_headers); + let mut count = 0; + let mut data = Bytes::new(); + while number < last && number > 1 && count < max_count { + match io.chain.block_header_at(number) { + Some(mut hdr) => { + data.append(&mut hdr); + count += 1; + } + None => {} + } + number += (if reverse { -(skip + 1) } else { skip + 1 }) as BlockNumber; + } + let mut rlp = RlpStream::new_list(count as usize); + rlp.append_raw(&data, count as usize); + io.network.respond(BLOCK_HEADERS_PACKET, rlp.out()); + } + + fn return_block_bodies(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let mut count = r.item_count(); + if count == 0 { + debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); + return; + } + count = min(count, MAX_BODIES_TO_SEND); + let mut added = 0usize; + let mut data = Bytes::new(); + for i in 0..count { + match io.chain.block_body(&r.val_at::(i)) { + Some(mut hdr) => { + data.append(&mut hdr); + added += 1; + } + None => {} + } + } + let mut rlp = RlpStream::new_list(added); + rlp.append_raw(&data, added); + io.network.respond(BLOCK_BODIES_PACKET, rlp.out()); + } + + fn return_node_data(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let mut count = r.item_count(); + if count == 0 { + debug!(target: "sync", "Empty GetNodeData request, ignoring."); + return; + } + count = min(count, MAX_NODE_DATA_TO_SEND); + let mut added = 0usize; + let mut data = Bytes::new(); + for i in 0..count { + match io.chain.state_data(&r.val_at::(i)) { + Some(mut hdr) => { + data.append(&mut hdr); + added += 1; + } + None => {} + } + } + let mut rlp = RlpStream::new_list(added); + rlp.append_raw(&data, added); + io.network.respond(NODE_DATA_PACKET, rlp.out()); + } + + fn return_receipts(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + let mut count = r.item_count(); + if count == 0 { + debug!(target: "sync", "Empty GetReceipts request, ignoring."); + return; + } + count = min(count, MAX_RECEIPTS_TO_SEND); + let mut added = 0usize; + let mut data = Bytes::new(); + for i in 0..count { + match io.chain.block_receipts(&r.val_at::(i)) { + Some(mut hdr) => { + data.append(&mut hdr); + added += 1; + } + None => {} + } + } + let mut rlp = RlpStream::new_list(added); + rlp.append_raw(&data, added); + io.network.respond(RECEIPTS_PACKET, rlp.out()); + } + + pub fn on_packet(&mut self, io: &mut SyncIo, peer: &PeerId, packet_id: u8, data: &[u8]) { + let rlp = Rlp::new(data); + match packet_id { + STATUS_PACKET => self.on_peer_status(io, peer, &rlp), + TRANSACTIONS_PACKET => self.on_peer_transactions(io, peer, &rlp), + GET_BLOCK_HEADERS_PACKET => self.return_block_headers(io, peer, &rlp), + BLOCK_HEADERS_PACKET => self.on_peer_block_headers(io, peer, &rlp), + GET_BLOCK_BODIES_PACKET => self.return_block_bodies(io, peer, &rlp), + BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp), + NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp), + NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), + GET_NODE_DATA_PACKET => self.return_node_data(io, peer, &rlp), + GET_RECEIPTS_PACKET => self.return_receipts(io, peer, &rlp), + _ => debug!(target: "sync", "Unkown packet {}", packet_id) + } + } } pub trait ToUsize { @@ -426,7 +815,7 @@ pub trait ToUsize { } pub trait FromUsize { - fn from(s: usize) -> Self; + fn from_usize(s: usize) -> Self; } impl ToUsize for BlockNumber { @@ -436,7 +825,7 @@ impl ToUsize for BlockNumber { } impl FromUsize for BlockNumber { - fn from(s: usize) -> BlockNumber { + fn from_usize(s: usize) -> BlockNumber { s as BlockNumber } } @@ -444,6 +833,7 @@ impl FromUsize for BlockNumber { pub trait RangeCollection { fn have_item(&self, key: &K) -> bool; fn find_item(&self, key: &K) -> Option<&V>; + fn get_tail(&mut self, key: &K) -> Range; fn remove_head(&mut self, start: &K); fn remove_tail(&mut self, start: &K); fn insert_item(&mut self, key: K, value: V); @@ -454,7 +844,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { Ok(_) => true, Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from(v.len())) > *key, + Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from_usize(v.len())) > *key, _ => false }, } @@ -464,12 +854,28 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { Ok(index) => self.get(index).unwrap().1.get(0), Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from(v.len())) > *key => v.get((*key - *k).to_usize()), + Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => v.get((*key - *k).to_usize()), _ => None }, } } + /// Get a range of elements from start till the end of the range + fn get_tail(&mut self, key: &K) -> Range { + let kv = *key; + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => kv..(kv + FromUsize::from_usize(self[index].1.len())), + Err(index) => { + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { + kv..(*k + FromUsize::from_usize(v.len())) + } + _ => kv..kv + } + }, + } + } /// Remove element key and following elements in the same range fn remove_tail(&mut self, key: &K) { match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { @@ -477,7 +883,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Err(index) =>{ let mut empty = false; match self.get_mut(index + 1) { - Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from(v.len())) > *key => { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { v.truncate((*key - *k).to_usize()); empty = v.is_empty(); } @@ -492,21 +898,21 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + /// Remove range elements up to key fn remove_head(&mut self, key: &K) { - if *key == FromUsize::from(0) { + if *key == FromUsize::from_usize(0) { return } - let prev = *key - FromUsize::from(1); + let prev = *key - FromUsize::from_usize(1); match self.binary_search_by(|&(k, _)| k.cmp(&prev).reverse()) { Ok(index) => { self.remove(index); }, Err(index) => { let mut empty = false; match self.get_mut(index + 1) { - Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from(v.len())) > *key => { + Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from_usize(v.len())) > *key => { let head = v.split_off((*key - *k).to_usize()); empty = head.is_empty(); let removed = ::std::mem::replace(v, head); - let new_k = *k - FromUsize::from(removed.len()); + let new_k = *k - FromUsize::from_usize(removed.len()); ::std::mem::replace(k, new_k); } _ => {} @@ -529,7 +935,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + lower += 1; let mut to_remove: Option = None; - if lower < self.len() && self[lower].0 + FromUsize::from(self[lower].1.len()) == key { + if lower < self.len() && self[lower].0 + FromUsize::from_usize(self[lower].1.len()) == key { // extend into existing chunk self[lower].1.push(value); } @@ -545,7 +951,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + let (mut next, mut inserted) = self.split_at_mut(lower); let mut next = next.last_mut().unwrap(); let mut inserted = inserted.first_mut().unwrap(); - if next.0 == key + FromUsize::from(1) + if next.0 == key + FromUsize::from_usize(1) { inserted.1.append(&mut next.1); to_remove = Some(lower - 1); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 29e61b7d0..3ae2e7bc5 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,6 +1,7 @@ use std::sync::{Arc}; use eth::{BlockChainClient}; -use util::network::{Error as NetworkError, ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId}; +use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId}; +use sync::chain::{ChainSync, SyncIo}; mod chain; @@ -11,7 +12,8 @@ pub fn new(service: &mut NetworkService, eth_cleint: Arc) { struct EthSync { idle: bool, - chain: Arc + chain: Arc, + sync: ChainSync } impl ProtocolHandler for EthSync { @@ -20,12 +22,15 @@ impl ProtocolHandler for EthSync { } fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { + self.sync.on_packet(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer, packet_id, data); } fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { + self.sync.on_peer_connected(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); } fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { + self.sync.on_peer_aborting(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); } fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { From 9087cc798b5b99cd7439ce4a8e7ae9f9cb73a068 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 25 Dec 2015 14:55:55 +0100 Subject: [PATCH 03/27] sync refactoring; range tests --- .gitignore | 3 + src/bin/client.rs | 2 +- src/eth.rs | 9 +- src/sync/chain.rs | 334 +++++++++++------------------------ src/sync/mod.rs | 40 ++++- src/sync/range_collection | 136 ++++++++++++++ src/sync/range_collection.rs | 244 +++++++++++++++++++++++++ src/sync/tests.rs | 13 ++ 8 files changed, 543 insertions(+), 238 deletions(-) create mode 100644 src/sync/range_collection create mode 100644 src/sync/range_collection.rs create mode 100644 src/sync/tests.rs diff --git a/.gitignore b/.gitignore index 5fc1b92c5..8b4a3b588 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ Cargo.lock # mac stuff .DS_Store + +# gdb files +.gdb_history diff --git a/src/bin/client.rs b/src/bin/client.rs index 5bc2adaab..0af81e47f 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -5,7 +5,7 @@ use util::network::{NetworkService}; fn main() { - let mut service = NetworkService::start().unwrap(); + let _service = NetworkService::start().unwrap(); loop { let mut cmd = String::new(); stdin().read_line(&mut cmd).unwrap(); diff --git a/src/eth.rs b/src/eth.rs index 3a8853bac..e82a899a0 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -31,7 +31,10 @@ pub struct BlockChainInfo { pub last_block_number: BlockNumber } -pub struct BlockQueueStats; +pub struct BlockQueueStatus { + pub full: bool, +} + pub struct TreeRoute; pub type BlockNumber = u32; @@ -50,7 +53,7 @@ pub trait BlockChainClient : Sync { fn state_data(&self, h: &H256) -> Option; fn block_receipts(&self, h: &H256) -> Option; fn import_block(&mut self, b: &[u8]) -> ImportResult; - fn queue_stats(&self) -> BlockQueueStats; - fn clear_queue(&mut self) -> BlockQueueStats; + fn queue_status(&self) -> BlockQueueStatus; + fn clear_queue(&mut self); fn info(&self) -> BlockChainInfo; } diff --git a/src/sync/chain.rs b/src/sync/chain.rs index c14abb89a..7f6b99ccb 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -1,6 +1,5 @@ use std::collections::{HashSet, HashMap}; use std::cmp::{min, max}; -use std::ops::{Add, Sub, Range}; use std::mem::{replace}; use util::network::{PeerId, HandlerIo, PacketId}; use util::hash::{H256}; @@ -10,6 +9,7 @@ use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp use util::rlp::rlptraits::{Stream, View}; use util::sha3::Hashable; use eth::{BlockNumber, BlockChainClient, BlockHeader, BlockStatus, QueueStatus, ImportResult}; +use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; pub struct SyncIo<'s, 'h> where 'h:'s { network: &'s mut HandlerIo<'h>, @@ -28,11 +28,25 @@ impl<'s, 'h> SyncIo<'s, 'h> { } } +impl ToUsize for BlockNumber { + fn to_usize(&self) -> usize { + *self as usize + } +} + +impl FromUsize for BlockNumber { + fn from_usize(s: usize) -> BlockNumber { + s as BlockNumber + } +} + const PROTOCOL_VERSION: u8 = 63u8; const MAX_BODIES_TO_SEND: usize = 256; const MAX_HEADERS_TO_SEND: usize = 1024; const MAX_NODE_DATA_TO_SEND: usize = 1024; const MAX_RECEIPTS_TO_SEND: usize = 1024; +const MAX_HEADERS_TO_REQUEST: usize = 1024; +const MAX_BODIES_TO_REQUEST: usize = 256; const STATUS_PACKET: u8 = 0x00; const NEW_BLOCK_HASHES_PACKET: u8 = 0x01; @@ -79,13 +93,13 @@ pub enum SyncState { } pub struct SyncStatus { - state: SyncState, - protocol_version: u8, - start_block_number: BlockNumber, - current_block_number: BlockNumber, - highest_block_number: BlockNumber, - blocks_total: usize, - blocks_received: usize + pub state: SyncState, + pub protocol_version: u8, + pub start_block_number: BlockNumber, + pub current_block_number: BlockNumber, + pub highest_block_number: BlockNumber, + pub blocks_total: usize, + pub blocks_received: usize } #[derive(PartialEq, Eq, Debug)] @@ -138,8 +152,8 @@ pub struct ChainSync { impl ChainSync { - pub fn new(io: &mut SyncIo) -> ChainSync { - let mut sync = ChainSync { + pub fn new() -> ChainSync { + ChainSync { state: SyncState::NotSynced, starting_block: 0, highest_block: 0, @@ -152,9 +166,7 @@ impl ChainSync { last_imported_block: 0, syncing_difficulty: U256::from(0u64), have_common_block: false - }; - sync.restart(io); - sync + } } /// @returns Synchonization status @@ -177,8 +189,8 @@ impl ChainSync { } /// @returns true is Sync is in progress - pub fn is_syncing(&self) { - self.state != SyncState::Idle; + pub fn is_syncing(&self) -> bool { + self.state != SyncState::Idle } fn reset(&mut self) { @@ -217,6 +229,9 @@ impl ChainSync { asking: PeerAsking::Nothing, asking_blocks: Vec::new(), }; + + trace!(target: "sync", "New peer (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer.protocol_version, peer.network_id, peer.difficulty, peer.latest, peer.genesis); + let old = self.peers.insert(peer_id.clone(), peer); if old.is_some() { panic!("ChainSync: new peer already exists"); @@ -423,6 +438,7 @@ impl ChainSync { /// Enter waiting state fn pause_sync(&mut self) { + trace!(target: "sync", "Block queue full, pausing sync"); self.state = SyncState::Waiting; } @@ -431,12 +447,12 @@ impl ChainSync { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); if peer.asking != PeerAsking::Nothing { - debug!(target: "sync", "Can't sync with this peer - outstanding asks."); + trace!(target: "sync", "Can't sync with this peer - outstanding asks."); return; } if self.state == SyncState::Waiting { - debug!(target: "sync", "Waiting for block queue"); + trace!(target: "sync", "Waiting for block queue"); return; } (peer.latest.clone(), peer.difficulty.clone()) @@ -459,24 +475,30 @@ impl ChainSync { fn request_blocks(&mut self, io: &mut SyncIo, peer_id: &PeerId) { self.clear_peer_download(peer_id); + + if io.chain.queue_status().full { + self.pause_sync(); + return; + } + // check to see if we need to download any block bodies first let mut needed_bodies: Vec = Vec::new(); let mut needed_numbers: Vec = Vec::new(); - let mut index = 0usize; - if self.have_common_block && !self.headers.is_empty() && self.headers.last().unwrap().0 == self.last_imported_block + 1 { - let mut header = self.headers.len() - 1; - while header != 0 && needed_bodies.len() < 1024 && index < self.headers[header].1.len() { - let block = self.headers[header].0 + index as BlockNumber; - if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { - needed_bodies.push(self.headers[header].1[index].hash.clone()); - needed_numbers.push(block); - self.downloading_bodies.insert(block); + if self.have_common_block && !self.headers.is_empty() && self.headers.range_iter().next().unwrap().0 == self.last_imported_block + 1 { + for (start, ref items) in self.headers.range_iter() { + if needed_bodies.len() >= MAX_BODIES_TO_REQUEST { + break; } - index += 1; - if index >= self.headers[header].1.len() { - index = 0; - header -= 1; + let mut index: BlockNumber = 0; + while index != items.len() as BlockNumber && needed_bodies.len() < MAX_BODIES_TO_REQUEST { + let block = start + index; + if !self.downloading_bodies.contains(&block) && !self.bodies.have_item(&block) { + needed_bodies.push(items[index as usize].hash.clone()); + needed_numbers.push(block); + self.downloading_bodies.insert(block); + } + index += 1; } } } @@ -491,44 +513,43 @@ impl ChainSync { // download backwards until common block is found 1 header at a time start = io.chain.info().last_block_number as usize; if !self.headers.is_empty() { - start = min(start, self.headers.last().unwrap().0 as usize - 1); + start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); } if start <= 1 { self.have_common_block = true; //reached genesis } } if self.have_common_block { - start = self.last_imported_block as usize + 1; - let mut next = self.headers.len() - 1; - let mut count = 0usize; - if !self.headers.is_empty() && start >= self.headers.last().unwrap().0 as usize { - start = self.headers.last().unwrap().0 as usize + self.headers.last().unwrap().1.len(); - next -=1; - } - while count == 0 && next != 0 { - count = min(1024, self.headers[next].0 as usize - start); - while count > 0 && self.downloading_headers.contains(&(start as BlockNumber)) { - start +=1; - count -=1; - } - } + + + let mut headers: Vec = Vec::new(); - for block in start..(start + count) { - if !self.downloading_headers.contains(&(block as BlockNumber)) { - headers.push(block as BlockNumber); - self.downloading_headers.insert(block as BlockNumber); + let mut prev = self.last_imported_block + 1; + for (start, ref items) in self.headers.range_iter() { + if headers.len() >= MAX_HEADERS_TO_REQUEST { + break; } + if start > prev { + continue; + } + let mut index = 0; + while index != items.len() as BlockNumber && headers.len() < MAX_BODIES_TO_REQUEST { + let block = prev + index; + if !self.downloading_headers.contains(&(block as BlockNumber)) { + headers.push(block as BlockNumber); + self.downloading_headers.insert(block as BlockNumber); + } + index += 1; + } + prev = start + items.len() as BlockNumber; } - count = self.headers.len(); - if count > 0 { + + if !headers.is_empty() { + let count = headers.len(); replace(&mut self.peers.get_mut(peer_id).unwrap().asking_blocks, headers); assert!(!self.headers.have_item(&(start as BlockNumber))); self.request_headers_by_number(io, peer_id, start as BlockNumber, count, 0, false); } - else if start >= (self.headers[next].0 as usize) { - start = self.headers[next].0 as usize + self.headers[next].1.len(); - next -=1; - } } else { self.request_headers_by_number(io, peer_id, start as BlockNumber, 1, 0, false); @@ -553,8 +574,8 @@ impl ChainSync { let mut restart = false; // merge headers and bodies { - let mut headers = self.headers.last().unwrap(); - let mut bodies = self.bodies.last().unwrap(); + let headers = self.headers.range_iter().next().unwrap(); + let bodies = self.bodies.range_iter().next().unwrap(); if headers.0 != bodies.0 || headers.0 != self.last_imported_block + 1 { return; } @@ -593,8 +614,8 @@ impl ChainSync { return; } - self.headers.remove_head(&self.last_imported_block); - self.bodies.remove_head(&self.last_imported_block); + self.headers.remove_head(&(self.last_imported_block + 1)); + self.bodies.remove_head(&(self.last_imported_block + 1)); if self.headers.is_empty() { assert!(self.bodies.is_empty()); @@ -645,12 +666,12 @@ impl ChainSync { for h in hashes { rlp.append(&h); } - self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_BODIES_PACKET, rlp.out()); + self.send_request(sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); } fn send_request(&mut self, sync: &mut SyncIo, peer_id: &PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { { - let mut peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); if peer.asking != PeerAsking::Nothing { warn!(target:"sync", "Asking {:?} while requesting {:?}", asking, peer.asking); } @@ -668,7 +689,7 @@ impl ChainSync { } } - fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: &PeerId, _r: &Rlp) { } fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { @@ -682,13 +703,12 @@ impl ChainSync { self.send_request(io, peer_id, PeerAsking::State, STATUS_PACKET, packet.out()); } - fn return_block_headers(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn return_block_headers(&self, io: &mut SyncIo, r: &Rlp) { // Packet layout: // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] let max_headers: usize = r.val_at(1); let skip: usize = r.val_at(2); let reverse: bool = r.val_at(3); - let mut packet = RlpStream::new(); let last = io.chain.info().last_block_number; let mut number = if r.at(0).size() == 32 { // id is a hash @@ -716,14 +736,20 @@ impl ChainSync { } None => {} } - number += (if reverse { -(skip + 1) } else { skip + 1 }) as BlockNumber; + if reverse { + number -= (skip + 1) as BlockNumber; + } + else { + number += (skip + 1) as BlockNumber; + } } let mut rlp = RlpStream::new_list(count as usize); rlp.append_raw(&data, count as usize); - io.network.respond(BLOCK_HEADERS_PACKET, rlp.out()); + io.network.respond(BLOCK_HEADERS_PACKET, rlp.out()).unwrap_or_else(|e| + debug!(target: "sync", "Error sending headers: {:?}", e)); } - fn return_block_bodies(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn return_block_bodies(&self, io: &mut SyncIo, r: &Rlp) { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); @@ -743,10 +769,11 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(BLOCK_BODIES_PACKET, rlp.out()); + io.network.respond(BLOCK_BODIES_PACKET, rlp.out()).unwrap_or_else(|e| + debug!(target: "sync", "Error sending headers: {:?}", e)); } - fn return_node_data(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn return_node_data(&self, io: &mut SyncIo, r: &Rlp) { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetNodeData request, ignoring."); @@ -766,10 +793,11 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(NODE_DATA_PACKET, rlp.out()); + io.network.respond(NODE_DATA_PACKET, rlp.out()).unwrap_or_else(|e| + debug!(target: "sync", "Error sending headers: {:?}", e)); } - fn return_receipts(&self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn return_receipts(&self, io: &mut SyncIo, r: &Rlp) { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetReceipts request, ignoring."); @@ -789,7 +817,8 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(RECEIPTS_PACKET, rlp.out()); + io.network.respond(RECEIPTS_PACKET, rlp.out()).unwrap_or_else(|e| + debug!(target: "sync", "Error sending headers: {:?}", e)); } pub fn on_packet(&mut self, io: &mut SyncIo, peer: &PeerId, packet_id: u8, data: &[u8]) { @@ -797,170 +826,19 @@ impl ChainSync { match packet_id { STATUS_PACKET => self.on_peer_status(io, peer, &rlp), TRANSACTIONS_PACKET => self.on_peer_transactions(io, peer, &rlp), - GET_BLOCK_HEADERS_PACKET => self.return_block_headers(io, peer, &rlp), + GET_BLOCK_HEADERS_PACKET => self.return_block_headers(io, &rlp), BLOCK_HEADERS_PACKET => self.on_peer_block_headers(io, peer, &rlp), - GET_BLOCK_BODIES_PACKET => self.return_block_bodies(io, peer, &rlp), + GET_BLOCK_BODIES_PACKET => self.return_block_bodies(io, &rlp), BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp), NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp), NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), - GET_NODE_DATA_PACKET => self.return_node_data(io, peer, &rlp), - GET_RECEIPTS_PACKET => self.return_receipts(io, peer, &rlp), + GET_NODE_DATA_PACKET => self.return_node_data(io, &rlp), + GET_RECEIPTS_PACKET => self.return_receipts(io, &rlp), _ => debug!(target: "sync", "Unkown packet {}", packet_id) } } -} -pub trait ToUsize { - fn to_usize(&self) -> usize; -} - -pub trait FromUsize { - fn from_usize(s: usize) -> Self; -} - -impl ToUsize for BlockNumber { - fn to_usize(&self) -> usize { - *self as usize + pub fn maintain_sync(&mut self, _io: &mut SyncIo) { } } -impl FromUsize for BlockNumber { - fn from_usize(s: usize) -> BlockNumber { - s as BlockNumber - } -} - -pub trait RangeCollection { - fn have_item(&self, key: &K) -> bool; - fn find_item(&self, key: &K) -> Option<&V>; - fn get_tail(&mut self, key: &K) -> Range; - fn remove_head(&mut self, start: &K); - fn remove_tail(&mut self, start: &K); - fn insert_item(&mut self, key: K, value: V); -} - -impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Add + Sub + Copy + FromUsize + ToUsize { - fn have_item(&self, key: &K) -> bool { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(_) => true, - Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from_usize(v.len())) > *key, - _ => false - }, - } - } - - fn find_item(&self, key: &K) -> Option<&V> { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => self.get(index).unwrap().1.get(0), - Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => v.get((*key - *k).to_usize()), - _ => None - }, - } - } - - /// Get a range of elements from start till the end of the range - fn get_tail(&mut self, key: &K) -> Range { - let kv = *key; - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => kv..(kv + FromUsize::from_usize(self[index].1.len())), - Err(index) => { - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { - kv..(*k + FromUsize::from_usize(v.len())) - } - _ => kv..kv - } - }, - } - } - /// Remove element key and following elements in the same range - fn remove_tail(&mut self, key: &K) { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => { self.remove(index); }, - Err(index) =>{ - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { - v.truncate((*key - *k).to_usize()); - empty = v.is_empty(); - } - _ => {} - } - if empty { - self.remove(index + 1); - } - }, - } - } - - /// Remove range elements up to key - fn remove_head(&mut self, key: &K) { - if *key == FromUsize::from_usize(0) { - return - } - - let prev = *key - FromUsize::from_usize(1); - match self.binary_search_by(|&(k, _)| k.cmp(&prev).reverse()) { - Ok(index) => { self.remove(index); }, - Err(index) => { - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from_usize(v.len())) > *key => { - let head = v.split_off((*key - *k).to_usize()); - empty = head.is_empty(); - let removed = ::std::mem::replace(v, head); - let new_k = *k - FromUsize::from_usize(removed.len()); - ::std::mem::replace(k, new_k); - } - _ => {} - } - if empty { - self.remove(index + 1); - } - }, - } - } - - fn insert_item(&mut self, key: K, value: V) { - assert!(!self.have_item(&key)); - - let mut lower = match self.binary_search_by(|&(k, _)| k.cmp(&key).reverse()) { - Ok(index) => index, - Err(index) => index, - }; - - lower += 1; - - let mut to_remove: Option = None; - if lower < self.len() && self[lower].0 + FromUsize::from_usize(self[lower].1.len()) == key { - // extend into existing chunk - self[lower].1.push(value); - } - else { - // insert a new chunk - let mut range: Vec = vec![value]; - self.insert(lower, (key, range)); - }; - let next = lower - 1; - if next < self.len() - { - { - let (mut next, mut inserted) = self.split_at_mut(lower); - let mut next = next.last_mut().unwrap(); - let mut inserted = inserted.first_mut().unwrap(); - if next.0 == key + FromUsize::from_usize(1) - { - inserted.1.append(&mut next.1); - to_remove = Some(lower - 1); - } - } - - if let Some(r) = to_remove { - self.remove(r); - } - } - } -} diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 3ae2e7bc5..3ec9c26ce 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -4,21 +4,48 @@ use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, Peer use sync::chain::{ChainSync, SyncIo}; mod chain; +mod range_collection; + +#[cfg(test)] +mod tests; -pub fn new(service: &mut NetworkService, eth_cleint: Arc) { - +pub fn new(_service: &mut NetworkService, eth_client: Arc) -> EthSync { + EthSync { + chain: eth_client, + sync: ChainSync::new(), + } } -struct EthSync { - idle: bool, +pub struct EthSync { chain: Arc, sync: ChainSync } +pub use self::chain::SyncStatus; + +impl EthSync { + pub fn is_syncing(&self) -> bool { + self.sync.is_syncing() + } + + pub fn status(&self) -> SyncStatus { + self.sync.status() + } + + pub fn stop_network(&mut self, io: &mut HandlerIo) { + self.sync.abort(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + } + + pub fn start_network(&mut self, io: &mut HandlerIo) { + self.sync.restart(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + } +} + impl ProtocolHandler for EthSync { fn initialize(&mut self, io: &mut HandlerIo) { - io.register_timer(1000); + self.sync.restart(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + io.register_timer(1000).unwrap(); } fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { @@ -33,7 +60,8 @@ impl ProtocolHandler for EthSync { self.sync.on_peer_aborting(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); } - fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { + fn timeout(&mut self, io: &mut HandlerIo, _timer: TimerToken) { + self.sync.maintain_sync(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } } diff --git a/src/sync/range_collection b/src/sync/range_collection new file mode 100644 index 000000000..cc1ccdf01 --- /dev/null +++ b/src/sync/range_collection @@ -0,0 +1,136 @@ + +pub trait RangeCollection { + fn have_item(&self, key: &K) -> bool; + fn find_item(&self, key: &K) -> Option<&V>; + fn get_tail(&mut self, key: &K) -> Range; + fn remove_head(&mut self, start: &K); + fn remove_tail(&mut self, start: &K); + fn insert_item(&mut self, key: K, value: V); +} + +impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Add + Sub + Copy + FromUsize + ToUsize { + fn have_item(&self, key: &K) -> bool { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(_) => true, + Err(index) => match self.get(index + 1) { + Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from_usize(v.len())) > *key, + _ => false + }, + } + } + + fn find_item(&self, key: &K) -> Option<&V> { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => self.get(index).unwrap().1.get(0), + Err(index) => match self.get(index + 1) { + Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => v.get((*key - *k).to_usize()), + _ => None + }, + } + } + + /// Get a range of elements from start till the end of the range + fn get_tail(&mut self, key: &K) -> Range { + let kv = *key; + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => kv..(kv + FromUsize::from_usize(self[index].1.len())), + Err(index) => { + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { + kv..(*k + FromUsize::from_usize(v.len())) + } + _ => kv..kv + } + }, + } + } + /// Remove element key and following elements in the same range + fn remove_tail(&mut self, key: &K) { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => { self.remove(index); }, + Err(index) =>{ + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { + v.truncate((*key - *k).to_usize()); + empty = v.is_empty(); + } + _ => {} + } + if empty { + self.remove(index + 1); + } + }, + } + } + + /// Remove range elements up to key + fn remove_head(&mut self, key: &K) { + if *key == FromUsize::from_usize(0) { + return + } + + let prev = *key - FromUsize::from_usize(1); + match self.binary_search_by(|&(k, _)| k.cmp(&prev).reverse()) { + Ok(index) => { self.remove(index); }, + Err(index) => { + let mut empty = false; + match self.get_mut(index + 1) { + Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from_usize(v.len())) > *key => { + let head = v.split_off((*key - *k).to_usize()); + empty = head.is_empty(); + let removed = ::std::mem::replace(v, head); + let new_k = *k - FromUsize::from_usize(removed.len()); + ::std::mem::replace(k, new_k); + } + _ => {} + } + if empty { + self.remove(index + 1); + } + }, + } + } + + fn insert_item(&mut self, key: K, value: V) { + assert!(!self.have_item(&key)); + + let mut lower = match self.binary_search_by(|&(k, _)| k.cmp(&key).reverse()) { + Ok(index) => index, + Err(index) => index, + }; + + lower += 1; + + let mut to_remove: Option = None; + if lower < self.len() && self[lower].0 + FromUsize::from_usize(self[lower].1.len()) == key { + // extend into existing chunk + self[lower].1.push(value); + } + else { + // insert a new chunk + let mut range: Vec = vec![value]; + self.insert(lower, (key, range)); + }; + let next = lower - 1; + if next < self.len() + { + { + let (mut next, mut inserted) = self.split_at_mut(lower); + let mut next = next.last_mut().unwrap(); + let mut inserted = inserted.first_mut().unwrap(); + if next.0 == key + FromUsize::from_usize(1) + { + inserted.1.append(&mut next.1); + to_remove = Some(lower - 1); + } + } + + if let Some(r) = to_remove { + self.remove(r); + } + } + } +} + diff --git a/src/sync/range_collection.rs b/src/sync/range_collection.rs new file mode 100644 index 000000000..5d19d0fd8 --- /dev/null +++ b/src/sync/range_collection.rs @@ -0,0 +1,244 @@ +use std::ops::{Add, Sub, Range}; + +pub trait ToUsize { + fn to_usize(&self) -> usize; +} + +pub trait FromUsize { + fn from_usize(s: usize) -> Self; +} + +pub trait RangeCollection { + fn have_item(&self, key: &K) -> bool; + fn find_item(&self, key: &K) -> Option<&V>; + fn get_tail(&mut self, key: &K) -> Range; + fn remove_head(&mut self, start: &K); + fn remove_tail(&mut self, start: &K); + fn insert_item(&mut self, key: K, value: V); + fn range_iter<'c>(&'c self) -> RangeIterator<'c, K, V>; +} + +pub struct RangeIterator<'c, K:'c, V:'c> { + range: usize, + collection: &'c Vec<(K, Vec)> +} + +impl<'c, K:'c, V:'c> Iterator for RangeIterator<'c, K, V> where K: Add + FromUsize + ToUsize + Copy { + type Item = (K, &'c [V]); + // The 'Iterator' trait only requires the 'next' method to be defined. The + // return type is 'Option', 'None' is returned when the 'Iterator' is + // over, otherwise the next value is returned wrapped in 'Some' + fn next(&mut self) -> Option<(K, &'c [V])> { + if self.range > 0 { + self.range -= 1; + } + else { + return None; + } + match self.collection.get(self.range) { + Some(&(ref k, ref vec)) => { + Some((*k, &vec)) + }, + None => None + } + } +} + +impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Add + Sub + Copy + FromUsize + ToUsize { + fn range_iter<'c>(&'c self) -> RangeIterator<'c, K, V> { + RangeIterator { + range: self.len(), + collection: self + } + } + + fn have_item(&self, key: &K) -> bool { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(_) => true, + Err(index) => match self.get(index) { + Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from_usize(v.len())) > *key, + _ => false + }, + } + } + + fn find_item(&self, key: &K) -> Option<&V> { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => self.get(index).unwrap().1.get(0), + Err(index) => match self.get(index) { + Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => v.get((*key - *k).to_usize()), + _ => None + }, + } + } + + /// Get a range of elements from start till the end of the range + fn get_tail(&mut self, key: &K) -> Range { + let kv = *key; + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => kv..(kv + FromUsize::from_usize(self[index].1.len())), + Err(index) => { + match self.get_mut(index) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { + kv..(*k + FromUsize::from_usize(v.len())) + } + _ => kv..kv + } + }, + } + } + /// Remove element key and following elements in the same range + fn remove_tail(&mut self, key: &K) { + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(index) => { self.remove(index); }, + Err(index) =>{ + let mut empty = false; + match self.get_mut(index) { + Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { + v.truncate((*key - *k).to_usize()); + empty = v.is_empty(); + } + _ => {} + } + if empty { + self.remove(index); + } + }, + } + } + + /// Remove range elements up to key + fn remove_head(&mut self, key: &K) { + if *key == FromUsize::from_usize(0) { + return + } + + let prev = *key - FromUsize::from_usize(1); + match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { + Ok(_) => { }, //start of range, do nothing. + Err(index) => { + let mut empty = false; + match self.get_mut(index) { + Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from_usize(v.len())) > prev => { + let tail = v.split_off((*key - *k).to_usize()); + empty = tail.is_empty(); + let removed = ::std::mem::replace(v, tail); + let new_k = *k + FromUsize::from_usize(removed.len()); + ::std::mem::replace(k, new_k); + } + _ => {} + } + if empty { + self.remove(index); + } + }, + } + } + + fn insert_item(&mut self, key: K, value: V) { + assert!(!self.have_item(&key)); + + let lower = match self.binary_search_by(|&(k, _)| k.cmp(&key).reverse()) { + Ok(index) => index, + Err(index) => index, + }; + + let mut to_remove: Option = None; + if lower < self.len() && self[lower].0 + FromUsize::from_usize(self[lower].1.len()) == key { + // extend into existing chunk + self[lower].1.push(value); + } + else { + // insert a new chunk + let range: Vec = vec![value]; + self.insert(lower, (key, range)); + }; + if lower > 0 { + let next = lower - 1; + if next < self.len() + { + { + let (mut next, mut inserted) = self.split_at_mut(lower); + let mut next = next.last_mut().unwrap(); + let mut inserted = inserted.first_mut().unwrap(); + if next.0 == key + FromUsize::from_usize(1) + { + inserted.1.append(&mut next.1); + to_remove = Some(lower - 1); + } + } + + if let Some(r) = to_remove { + self.remove(r); + } + } + } + } +} + +#[test] +fn test_range() { + use std::cmp::{Ordering}; + + let mut ranges: Vec<(u32, Vec)> = Vec::new(); + assert_eq!(ranges.range_iter().next(), None); + assert_eq!(ranges.find_item(&1), None); + assert!(!ranges.have_item(&1)); + assert_eq!(ranges.get_tail(&0), 0..0); + + ranges.insert_item(17, 'q'); + assert_eq!(ranges.range_iter().cmp(vec![(17, &['q'][..])]), Ordering::Equal); + assert_eq!(ranges.find_item(&17), Some(&'q')); + assert!(ranges.have_item(&17)); + assert_eq!(ranges.get_tail(&17), 17..18); + + ranges.insert_item(18, 'r'); + assert_eq!(ranges.range_iter().cmp(vec![(17, &['q', 'r'][..])]), Ordering::Equal); + assert_eq!(ranges.find_item(&18), Some(&'r')); + assert!(ranges.have_item(&18)); + assert_eq!(ranges.get_tail(&17), 17..19); + + ranges.insert_item(16, 'p'); + assert_eq!(ranges.range_iter().cmp(vec![(16, &['p', 'q', 'r'][..])]), Ordering::Equal); + assert_eq!(ranges.find_item(&16), Some(&'p')); + assert_eq!(ranges.find_item(&17), Some(&'q')); + assert_eq!(ranges.find_item(&18), Some(&'r')); + assert!(ranges.have_item(&16)); + assert_eq!(ranges.get_tail(&17), 17..19); + + ranges.insert_item(2, 'b'); + assert_eq!(ranges.range_iter().cmp(vec![(2, &['b'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + assert_eq!(ranges.find_item(&2), Some(&'b')); + + ranges.insert_item(3, 'c'); + ranges.insert_item(4, 'd'); + assert_eq!(ranges.get_tail(&3), 3..5); + assert_eq!(ranges.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + + let mut r = ranges.clone(); + r.remove_head(&1); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_head(&2); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_head(&3); + assert_eq!(r.range_iter().cmp(vec![(3, &['c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_head(&10); + assert_eq!(r.range_iter().cmp(vec![(3, &['c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_head(&5); + assert_eq!(r.range_iter().cmp(vec![(16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_head(&19); + assert_eq!(r.range_iter().next(), None); + + let mut r = ranges.clone(); + r.remove_tail(&20); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); + r.remove_tail(&17); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..]), (16, &['p'][..])]), Ordering::Equal); + r.remove_tail(&16); + assert_eq!(r.range_iter().cmp(vec![(2, &['b', 'c', 'd'][..])]), Ordering::Equal); + r.remove_tail(&3); + assert_eq!(r.range_iter().cmp(vec![(2, &['b'][..])]), Ordering::Equal); + r.remove_tail(&2); + assert_eq!(r.range_iter().next(), None); +} + diff --git a/src/sync/tests.rs b/src/sync/tests.rs new file mode 100644 index 000000000..a51bf1543 --- /dev/null +++ b/src/sync/tests.rs @@ -0,0 +1,13 @@ +use std::collections::HashMap; +use util::bytes::Bytes; +use util::hash::H256; + +struct TestBlockChainClient { + blocks: Vec, + hashes: HashMap, +} + + +#[test] +fn full_sync() { +} From bf9667a206a10ce5d5172e0e411bf9cd50e99ee6 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 26 Dec 2015 15:47:07 +0100 Subject: [PATCH 04/27] sync tests --- src/blockchain.rs | 62 +++++------ src/eth.rs | 8 +- src/genesis.rs | 12 +- src/header.rs | 50 ++++++--- src/sync/chain.rs | 86 +++++++------- src/sync/mod.rs | 58 ++++++++-- src/sync/tests.rs | 278 +++++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 436 insertions(+), 118 deletions(-) diff --git a/src/blockchain.rs b/src/blockchain.rs index df729787a..f64d81189 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -19,7 +19,7 @@ use transaction::*; use views::*; /// Represents a tree route between `from` block and `to` block: -/// +/// /// - `blocks` - a vector of hashes of all blocks, ordered from `from` to `to`. /// /// - `ancestor` - best common ancestor of these blocks. @@ -59,7 +59,7 @@ impl BestBlock { } /// Structure providing fast access to blockchain data. -/// +/// /// **Does not do input data verification.** pub struct BlockChain { best_block: RefCell, @@ -80,7 +80,7 @@ pub struct BlockChain { impl BlockChain { /// Create new instance of blockchain from given Genesis - /// + /// /// ```rust /// extern crate ethcore_util as util; /// extern crate ethcore; @@ -90,7 +90,7 @@ impl BlockChain { /// use ethcore::blockchain::*; /// use util::hash::*; /// use util::uint::*; - /// + /// /// fn main() { /// let genesis = Genesis::new_frontier(); /// @@ -152,7 +152,7 @@ impl BlockChain { batch.put_extras(&header.number(), &hash); batch.put(b"best", &hash).unwrap(); bc.extras_db.write(batch).unwrap(); - + hash } }; @@ -168,44 +168,44 @@ impl BlockChain { } /// Returns a tree route between `from` and `to`, which is a tuple of: - /// + /// /// - a vector of hashes of all blocks, ordered from `from` to `to`. /// /// - common ancestor of these blocks. /// /// - an index where best common ancestor would be - /// + /// /// 1.) from newer to older - /// + /// /// - bc: `A1 -> A2 -> A3 -> A4 -> A5` /// - from: A5, to: A4 - /// - route: + /// - route: /// /// ```json /// { blocks: [A5], ancestor: A4, index: 1 } /// ``` - /// + /// /// 2.) from older to newer - /// + /// /// - bc: `A1 -> A2 -> A3 -> A4 -> A5` /// - from: A3, to: A4 - /// - route: - /// + /// - route: + /// /// ```json /// { blocks: [A4], ancestor: A3, index: 0 } /// ``` /// /// 3.) fork: /// - /// - bc: + /// - bc: /// /// ```text /// A1 -> A2 -> A3 -> A4 /// -> B3 -> B4 - /// ``` + /// ``` /// - from: B4, to: A4 - /// - route: - /// + /// - route: + /// /// ```json /// { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 } /// ``` @@ -302,7 +302,7 @@ impl BlockChain { // create views onto rlp let block = BlockView::new(bytes); let header = block.header_view(); - + // prepare variables let hash = block.sha3(); let mut parent_details = self.block_details(&header.parent_hash()).expect("Invalid parent hash."); @@ -317,7 +317,7 @@ impl BlockChain { parent: parent_hash.clone(), children: vec![] }; - + // prepare the batch let batch = WriteBatch::new(); @@ -333,7 +333,7 @@ impl BlockChain { return (batch, None); } - // if its new best block we need to make sure that all ancestors + // if its new best block we need to make sure that all ancestors // are moved to "canon chain" // find the route between old best block and the new one let best_hash = self.best_block_hash(); @@ -368,7 +368,7 @@ impl BlockChain { (batch, Some(best_block)) } - /// Returns true if the given block is known + /// Returns true if the given block is known /// (though not necessarily a part of the canon chain). pub fn is_known(&self, hash: &H256) -> bool { self.query_extras_exist(hash, &self.block_details) @@ -471,8 +471,8 @@ impl BlockChain { } } - fn query_extras(&self, hash: &K, cache: &RefCell>) -> Option where - T: Clone + Decodable + ExtrasIndexable, + fn query_extras(&self, hash: &K, cache: &RefCell>) -> Option where + T: Clone + Decodable + ExtrasIndexable, K: ExtrasSliceConvertable + Eq + Hash + Clone { { let read = cache.borrow(); @@ -489,7 +489,7 @@ impl BlockChain { }) } - fn query_extras_exist(&self, hash: &K, cache: &RefCell>) -> bool where + fn query_extras_exist(&self, hash: &K, cache: &RefCell>) -> bool where K: ExtrasSliceConvertable + Eq + Hash + Clone, T: ExtrasIndexable { { @@ -541,7 +541,7 @@ mod tests { dir.push(H32::random().hex()); let bc = BlockChain::new(&genesis, &dir); - + let genesis_hash = H256::from_str("3caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942").unwrap(); assert_eq!(bc.genesis_hash(), genesis_hash.clone()); @@ -549,7 +549,7 @@ mod tests { assert_eq!(bc.best_block_hash(), genesis_hash.clone()); assert_eq!(bc.block_hash(&U256::from(0u8)), Some(genesis_hash.clone())); assert_eq!(bc.block_hash(&U256::from(1u8)), None); - + let first = "f90285f90219a03caa2203f3d7c136c0295ed128a7d31cea520b1ca5e27afe17d0853331798942a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0bac6177a79e910c98d86ec31a09ae37ac2de15b754fd7bed1ba52362c49416bfa0d45893a296c1490a978e0bd321b5f2635d8280365c1fe9f693d65f233e791344a0c7778a7376099ee2e5c455791c1885b5c361b95713fddcbe32d97fd01334d296b90100000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000400000000000000000000000000000000000000000000000000000008302000001832fefd882560b845627cb99a00102030405060708091011121314151617181920212223242526272829303132a08ccb2837fb2923bd97e8f2d08ea32012d6e34be018c73e49a0f98843e8f47d5d88e53be49fec01012ef866f864800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d8785012a05f200801ba0cb088b8d2ff76a7b2c6616c9d02fb6b7a501afbf8b69d7180b09928a1b80b5e4a06448fe7476c606582039bb72a9f6f4b4fad18507b8dfbd00eebbe151cc573cd2c0".from_hex().unwrap(); @@ -610,22 +610,22 @@ mod tests { assert_eq!(r0_1.blocks, [b1_hash.clone()]); assert_eq!(r0_1.index, 0); - let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); + let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); assert_eq!(r0_2.ancestor, genesis_hash); assert_eq!(r0_2.blocks, [b1_hash.clone(), b2_hash.clone()]); assert_eq!(r0_2.index, 0); - let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); + let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); assert_eq!(r1_3a.ancestor, b1_hash); assert_eq!(r1_3a.blocks, [b2_hash.clone(), b3a_hash.clone()]); assert_eq!(r1_3a.index, 0); - let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); + let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); assert_eq!(r1_3b.ancestor, b1_hash); assert_eq!(r1_3b.blocks, [b2_hash.clone(), b3b_hash.clone()]); assert_eq!(r1_3b.index, 0); - let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); + let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); assert_eq!(r3a_3b.ancestor, b2_hash); assert_eq!(r3a_3b.blocks, [b3a_hash.clone(), b3b_hash.clone()]); assert_eq!(r3a_3b.index, 1); @@ -639,7 +639,7 @@ mod tests { assert_eq!(r2_0.ancestor, genesis_hash); assert_eq!(r2_0.blocks, [b2_hash.clone(), b1_hash.clone()]); assert_eq!(r2_0.index, 2); - + let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()); assert_eq!(r3a_1.ancestor, b1_hash); assert_eq!(r3a_1.blocks, [b3a_hash.clone(), b2_hash.clone()]); diff --git a/src/eth.rs b/src/eth.rs index e82a899a0..5eb7e9b6c 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -35,7 +35,7 @@ pub struct BlockQueueStatus { pub full: bool, } -pub struct TreeRoute; +pub type TreeRoute = ::blockchain::TreeRoute; pub type BlockNumber = u32; pub type BlockHeader = ::header::Header; @@ -46,9 +46,9 @@ pub trait BlockChainClient : Sync { fn block(&self, h: &H256) -> Option; fn block_status(&self, h: &H256) -> BlockStatus; fn block_header_at(&self, n: BlockNumber) -> Option; - fn block_body_at(&self, h: BlockNumber) -> Option; - fn block_at(&self, h: BlockNumber) -> Option; - fn block_status_at(&self, h: BlockNumber) -> BlockStatus; + fn block_body_at(&self, n: BlockNumber) -> Option; + fn block_at(&self, n: BlockNumber) -> Option; + fn block_status_at(&self, n: BlockNumber) -> BlockStatus; fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; fn state_data(&self, h: &H256) -> Option; fn block_receipts(&self, h: &H256) -> Option; diff --git a/src/genesis.rs b/src/genesis.rs index d4829a70e..3e95c098a 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,5 +1,6 @@ use std::io::Read; use std::str::FromStr; +use std::cell::Cell; use std::collections::HashMap; use rustc_serialize::base64::FromBase64; use rustc_serialize::json::Json; @@ -48,7 +49,7 @@ impl Genesis { Genesis { block: stream.out(), - state: state + state: state } } @@ -56,12 +57,12 @@ impl Genesis { fn load_genesis_json(json: &Json, state_root: &H256) -> (Header, HashMap) { // once we commit ourselves to some json parsing library (serde?) // move it to proper data structure - + let empty_list = RlpStream::new_list(0).out(); let empty_list_sha3 = empty_list.sha3(); let empty_data = encode(&""); let empty_data_sha3 = empty_data.sha3(); - + let header = Header { parent_hash: H256::from_str(&json["parentHash"].as_string().unwrap()[2..]).unwrap(), uncles_hash: empty_list_sha3.clone(), @@ -81,9 +82,10 @@ impl Genesis { let mixhash = H256::from_str(&json["mixhash"].as_string().unwrap()[2..]).unwrap(); let nonce = H64::from_str(&json["nonce"].as_string().unwrap()[2..]).unwrap(); vec![mixhash.to_vec(), nonce.to_vec()] - } + }, + hash: Cell::new(None) }; - + let mut state = HashMap::new(); let accounts = json["alloc"].as_object().expect("Missing genesis state"); for (address, acc) in accounts.iter() { diff --git a/src/header.rs b/src/header.rs index e161ac280..fdc51a549 100644 --- a/src/header.rs +++ b/src/header.rs @@ -1,4 +1,6 @@ +use std::cell::Cell; use util::hash::*; +use util::sha3::*; use util::bytes::*; use util::uint::*; use util::rlp::*; @@ -28,6 +30,8 @@ pub struct Header { pub difficulty: U256, pub seal: Vec, + + pub hash: Cell>, //TODO: make this private } impl Header { @@ -50,37 +54,49 @@ impl Header { difficulty: ZERO_U256.clone(), seal: vec![], + hash: Cell::new(None), } } pub fn hash(&self) -> H256 { - unimplemented!(); + let hash = self.hash.get(); + match hash { + Some(h) => h, + None => { + let mut stream = RlpStream::new(); + stream.append(self); + let h = stream.raw().sha3(); + self.hash.set(Some(h.clone())); + h + } + } } } impl Decodable for Header { fn decode(decoder: &D) -> Result where D: Decoder { - let d = try!(decoder.as_list()); + let r = decoder.as_rlp(); let mut blockheader = Header { - parent_hash: try!(Decodable::decode(&d[0])), - uncles_hash: try!(Decodable::decode(&d[1])), - author: try!(Decodable::decode(&d[2])), - state_root: try!(Decodable::decode(&d[3])), - transactions_root: try!(Decodable::decode(&d[4])), - receipts_root: try!(Decodable::decode(&d[5])), - log_bloom: try!(Decodable::decode(&d[6])), - difficulty: try!(Decodable::decode(&d[7])), - number: try!(Decodable::decode(&d[8])), - gas_limit: try!(Decodable::decode(&d[9])), - gas_used: try!(Decodable::decode(&d[10])), - timestamp: try!(Decodable::decode(&d[11])), - extra_data: try!(Decodable::decode(&d[12])), + parent_hash: try!(r.val_at(0)), + uncles_hash: try!(r.val_at(1)), + author: try!(r.val_at(2)), + state_root: try!(r.val_at(3)), + transactions_root: try!(r.val_at(4)), + receipts_root: try!(r.val_at(5)), + log_bloom: try!(r.val_at(6)), + difficulty: try!(r.val_at(7)), + number: try!(r.val_at(8)), + gas_limit: try!(r.val_at(9)), + gas_used: try!(r.val_at(10)), + timestamp: try!(r.val_at(11)), + extra_data: try!(r.val_at(12)), seal: vec![], + hash: Cell::new(Some(r.raw().sha3())) }; - for i in 13..d.len() { - blockheader.seal.push(try!(Decodable::decode(&d[i]))); + for i in 13..r.item_count() { + blockheader.seal.push(try!(r.val_at(i))) } Ok(blockheader) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 7f6b99ccb..0b921f552 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -1,7 +1,7 @@ use std::collections::{HashSet, HashMap}; use std::cmp::{min, max}; use std::mem::{replace}; -use util::network::{PeerId, HandlerIo, PacketId}; +use util::network::{PeerId, PacketId}; use util::hash::{H256}; use util::bytes::{Bytes}; use util::uint::{U256}; @@ -10,23 +10,7 @@ use util::rlp::rlptraits::{Stream, View}; use util::sha3::Hashable; use eth::{BlockNumber, BlockChainClient, BlockHeader, BlockStatus, QueueStatus, ImportResult}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; - -pub struct SyncIo<'s, 'h> where 'h:'s { - network: &'s mut HandlerIo<'h>, - chain: &'s mut BlockChainClient -} - -impl<'s, 'h> SyncIo<'s, 'h> { - pub fn new(network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient) -> SyncIo<'s,'h> { - SyncIo { - network: network, - chain: chain, - } - } - fn disable_peer(&mut self, peer_id: &PeerId) { - self.network.disable_peer(*peer_id); - } -} +use sync::{SyncIo}; impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { @@ -106,7 +90,6 @@ pub struct SyncStatus { enum PeerAsking { Nothing, - State, BlockHeaders, BlockBodies, } @@ -213,8 +196,8 @@ impl ChainSync { self.starting_block = 0; self.highest_block = 0; self.have_common_block = false; - io.chain.clear_queue(); - self.starting_block = io.chain.info().last_block_number; + io.chain().clear_queue(); + self.starting_block = io.chain().info().last_block_number; self.state = SyncState::NotSynced; } @@ -263,7 +246,7 @@ impl ChainSync { if number > self.highest_block { self.highest_block = number; } - match io.chain.block_status(&info.hash()) { + match io.chain().block_status(&info.hash()) { BlockStatus::InChain => { self.have_common_block = true; self.last_imported_block = number; @@ -285,7 +268,7 @@ impl ChainSync { } } let hdr = Header { - data: r.at(i).data().to_vec(), + data: r.at(i).raw().to_vec(), hash: info.hash(), parent: info.parent_hash, }; @@ -298,7 +281,7 @@ impl ChainSync { //empty body, just mark as downloaded let mut body_stream = RlpStream::new_list(2); body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); - body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); + body_stream.append_raw(&rlp::NULL_RLP, 1); self.bodies.insert_item(number, body_stream.out()); } else { @@ -327,8 +310,8 @@ impl ChainSync { for i in 0..item_count { let body: Rlp = r.at(i); let tx = body.at(0); - let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.data().to_vec()).collect()); //TODO: get rid of vectors here - let uncles = body.at(1).data().sha3(); + let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.raw().to_vec()).collect()); //TODO: get rid of vectors here + let uncles = body.at(1).raw().sha3(); let header_id = HeaderId { transactions_root: tx_root, uncles: uncles @@ -336,7 +319,7 @@ impl ChainSync { match self.header_ids.get(&header_id).map(|n| *n) { Some(n) => { self.header_ids.remove(&header_id); - self.bodies.insert_item(n, body.data().to_vec()); + self.bodies.insert_item(n, body.raw().to_vec()); } None => { debug!(target: "sync", "Ignored unknown block body"); @@ -351,9 +334,9 @@ impl ChainSync { fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { let block_rlp = r.at(0); let header_rlp = block_rlp.at(0); - let h = header_rlp.data().sha3(); + let h = header_rlp.raw().sha3(); - match io.chain.import_block(block_rlp.data()) { + match io.chain().import_block(block_rlp.raw()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "New block already in chain {:?}", h); }, @@ -388,7 +371,7 @@ impl ChainSync { let hashes = r.iter().map(|item| (item.val_at::(0), item.val_at::(1))); let mut max_height: U256 = From::from(0); for (h, d) in hashes { - match io.chain.block_status(&h) { + match io.chain().block_status(&h) { BlockStatus::InChain => { trace!(target: "sync", "New block hash already in chain {:?}", h); }, @@ -458,7 +441,7 @@ impl ChainSync { (peer.latest.clone(), peer.difficulty.clone()) }; - let td = io.chain.info().pending_total_difficulty; + let td = io.chain().info().pending_total_difficulty; let syncing_difficulty = max(self.syncing_difficulty, td); if force || peer_difficulty > syncing_difficulty { // start sync @@ -476,7 +459,7 @@ impl ChainSync { fn request_blocks(&mut self, io: &mut SyncIo, peer_id: &PeerId) { self.clear_peer_download(peer_id); - if io.chain.queue_status().full { + if io.chain().queue_status().full { self.pause_sync(); return; } @@ -511,7 +494,7 @@ impl ChainSync { let mut start = 0usize; if !self.have_common_block { // download backwards until common block is found 1 header at a time - start = io.chain.info().last_block_number as usize; + start = io.chain().info().last_block_number as usize; if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); } @@ -585,7 +568,7 @@ impl ChainSync { block_rlp.append_raw(&headers.1[i].data, 1); block_rlp.append_raw(&bodies.1[i], 2); let h = &headers.1[i].hash; - match io.chain.import_block(&block_rlp.out()) { + match io.chain().import_block(&block_rlp.out()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; @@ -676,7 +659,7 @@ impl ChainSync { warn!(target:"sync", "Asking {:?} while requesting {:?}", asking, peer.asking); } } - match sync.network.send(*peer_id, packet_id, packet) { + match sync.send(*peer_id, packet_id, packet) { Err(e) => { warn!(target:"sync", "Error sending request: {:?}", e); sync.disable_peer(peer_id); @@ -694,13 +677,20 @@ impl ChainSync { fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { let mut packet = RlpStream::new_list(5); - let chain = io.chain.info(); + let chain = io.chain().info(); packet.append(&(PROTOCOL_VERSION as u32)); packet.append(&0u32); //TODO: network id packet.append(&chain.total_difficulty); packet.append(&chain.last_block_hash); packet.append(&chain.genesis_hash); - self.send_request(io, peer_id, PeerAsking::State, STATUS_PACKET, packet.out()); + //TODO: handle timeout for status request + match io.send(*peer_id, STATUS_PACKET, packet.out()) { + Err(e) => { + warn!(target:"sync", "Error sending status request: {:?}", e); + io.disable_peer(peer_id); + } + Ok(_) => { } + } } fn return_block_headers(&self, io: &mut SyncIo, r: &Rlp) { @@ -709,12 +699,12 @@ impl ChainSync { let max_headers: usize = r.val_at(1); let skip: usize = r.val_at(2); let reverse: bool = r.val_at(3); - let last = io.chain.info().last_block_number; + let last = io.chain().info().last_block_number; let mut number = if r.at(0).size() == 32 { // id is a hash let hash: H256 = r.val_at(0); trace!(target: "sync", "GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); - match io.chain.block_header(&hash) { + match io.chain().block_header(&hash) { Some(hdr) => From::from(rlp::decode::(&hdr).number), None => last } @@ -729,7 +719,7 @@ impl ChainSync { let mut count = 0; let mut data = Bytes::new(); while number < last && number > 1 && count < max_count { - match io.chain.block_header_at(number) { + match io.chain().block_header_at(number) { Some(mut hdr) => { data.append(&mut hdr); count += 1; @@ -745,7 +735,7 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(count as usize); rlp.append_raw(&data, count as usize); - io.network.respond(BLOCK_HEADERS_PACKET, rlp.out()).unwrap_or_else(|e| + io.respond(BLOCK_HEADERS_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); } @@ -759,7 +749,7 @@ impl ChainSync { let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain.block_body(&r.val_at::(i)) { + match io.chain().block_body(&r.val_at::(i)) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -769,7 +759,7 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(BLOCK_BODIES_PACKET, rlp.out()).unwrap_or_else(|e| + io.respond(BLOCK_BODIES_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); } @@ -783,7 +773,7 @@ impl ChainSync { let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain.state_data(&r.val_at::(i)) { + match io.chain().state_data(&r.val_at::(i)) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -793,7 +783,7 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(NODE_DATA_PACKET, rlp.out()).unwrap_or_else(|e| + io.respond(NODE_DATA_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); } @@ -807,7 +797,7 @@ impl ChainSync { let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain.block_receipts(&r.val_at::(i)) { + match io.chain().block_receipts(&r.val_at::(i)) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -817,7 +807,7 @@ impl ChainSync { } let mut rlp = RlpStream::new_list(added); rlp.append_raw(&data, added); - io.network.respond(RECEIPTS_PACKET, rlp.out()).unwrap_or_else(|e| + io.respond(RECEIPTS_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); } @@ -834,7 +824,7 @@ impl ChainSync { NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), GET_NODE_DATA_PACKET => self.return_node_data(io, &rlp), GET_RECEIPTS_PACKET => self.return_receipts(io, &rlp), - _ => debug!(target: "sync", "Unkown packet {}", packet_id) + _ => debug!(target: "sync", "Unknown packet {}", packet_id) } } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 3ec9c26ce..8efb86a43 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,7 +1,7 @@ use std::sync::{Arc}; use eth::{BlockChainClient}; -use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId}; -use sync::chain::{ChainSync, SyncIo}; +use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, PacketId, Error as NetworkError}; +use sync::chain::{ChainSync}; mod chain; mod range_collection; @@ -9,7 +9,6 @@ mod range_collection; #[cfg(test)] mod tests; - pub fn new(_service: &mut NetworkService, eth_client: Arc) -> EthSync { EthSync { chain: eth_client, @@ -17,6 +16,45 @@ pub fn new(_service: &mut NetworkService, eth_client: Arc) -> Result<(), NetworkError>; + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; + fn chain<'s>(&'s mut self) -> &'s mut BlockChainClient; +} + +pub struct NetSyncIo<'s, 'h> where 'h:'s { + network: &'s mut HandlerIo<'h>, + chain: &'s mut BlockChainClient +} + +impl<'s, 'h> NetSyncIo<'s, 'h> { + pub fn new(network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient) -> NetSyncIo<'s,'h> { + NetSyncIo { + network: network, + chain: chain, + } + } +} + +impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { + fn disable_peer(&mut self, peer_id: &PeerId) { + self.network.disable_peer(*peer_id); + } + + fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ + self.network.respond(packet_id, data) + } + + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ + self.network.send(peer_id, packet_id, data) + } + + fn chain<'a>(&'a mut self) -> &'a mut BlockChainClient { + self.chain + } +} + pub struct EthSync { chain: Arc, sync: ChainSync @@ -34,34 +72,34 @@ impl EthSync { } pub fn stop_network(&mut self, io: &mut HandlerIo) { - self.sync.abort(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + self.sync.abort(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } pub fn start_network(&mut self, io: &mut HandlerIo) { - self.sync.restart(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + self.sync.restart(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } } impl ProtocolHandler for EthSync { fn initialize(&mut self, io: &mut HandlerIo) { - self.sync.restart(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + self.sync.restart(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); io.register_timer(1000).unwrap(); } fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { - self.sync.on_packet(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer, packet_id, data); + self.sync.on_packet(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer, packet_id, data); } fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { - self.sync.on_peer_connected(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); + self.sync.on_peer_connected(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); } fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { - self.sync.on_peer_aborting(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); + self.sync.on_peer_aborting(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap()), peer); } fn timeout(&mut self, io: &mut HandlerIo, _timer: TimerToken) { - self.sync.maintain_sync(&mut SyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); + self.sync.maintain_sync(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } } diff --git a/src/sync/tests.rs b/src/sync/tests.rs index a51bf1543..7be66ff2f 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -1,13 +1,285 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use util::bytes::Bytes; -use util::hash::H256; +use util::hash::{H256, FixedHash}; +use util::uint::{U256}; +use util::sha3::Hashable; +use util::rlp::{self, Rlp, RlpStream, View, Stream}; +use util::network::{PeerId, PacketId, Error as NetworkError}; +use eth::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, BlockHeader, QueueStatus}; +use sync::{SyncIo}; +use sync::chain::{ChainSync}; struct TestBlockChainClient { blocks: Vec, hashes: HashMap, + genesis_hash: H256, + last_hash: H256, + difficulty: U256 +} + +impl TestBlockChainClient { + fn new() -> TestBlockChainClient { + + let mut client = TestBlockChainClient { + blocks: Vec::new(), + hashes: HashMap::new(), + genesis_hash: H256::new(), + last_hash: H256::new(), + difficulty: From::from(0), + }; + client.add_blocks(1, true); // add genesis block + client.genesis_hash = client.last_hash; + client + } + + pub fn add_blocks(&mut self, count: usize, empty: bool) { + for n in self.blocks.len()..(self.blocks.len() + count) { + let mut header = BlockHeader::new(); + header.difficulty = From::from(n); + header.parent_hash = self.last_hash; + header.number = From::from(n); + let mut uncles = RlpStream::new_list(if empty {0} else {1}); + if !empty { + uncles.append(&H256::random()); + header.uncles_hash = uncles.raw().sha3(); + } + let mut rlp = RlpStream::new_list(3); + rlp.append(&header); + rlp.append_raw(uncles.raw(), 1); + rlp.append_raw(&rlp::NULL_RLP, 1); + self.import_block(rlp.raw()); + } + } +} + +impl BlockChainClient for TestBlockChainClient { + fn block_header(&self, h: &H256) -> Option { + self.hashes.get(h).and_then(|i| self.block_header_at(*i as BlockNumber)) + } + + fn block_body(&self, h: &H256) -> Option { + self.hashes.get(h).and_then(|i| self.block_body_at(*i as BlockNumber)) + } + + fn block(&self, h: &H256) -> Option { + self.hashes.get(h).map(|i| self.blocks[*i].clone()) + } + + fn block_status(&self, h: &H256) -> BlockStatus { + self.hashes.get(h).map(|i| self.block_status_at(*i as BlockNumber)).unwrap_or(BlockStatus::Unknown) + } + + fn block_header_at(&self, n: BlockNumber) -> Option { + self.blocks.get(n as usize).map(|r| Rlp::new(r).at(0).raw().to_vec()) + } + + fn block_body_at(&self, n: BlockNumber) -> Option { + self.blocks.get(n as usize).map(|r| { + let mut stream = RlpStream::new_list(2); + stream.append_raw(Rlp::new(&r).at(1).raw(), 1); + stream.append_raw(Rlp::new(&r).at(2).raw(), 1); + stream.out() + }) + } + + fn block_at(&self, n: BlockNumber) -> Option { + self.blocks.get(n as usize).map(|b| b.clone()) + } + + fn block_status_at(&self, n: BlockNumber) -> BlockStatus { + if (n as usize) < self.blocks.len() { + BlockStatus::InChain + } else { + BlockStatus::Unknown + } + } + + fn tree_route(&self, _from: &H256, _to: &H256) -> TreeRoute { + TreeRoute { + blocks: Vec::new(), + ancestor: H256::new(), + index: 0 + } + } + + fn state_data(&self, _h: &H256) -> Option { + None + } + + fn block_receipts(&self, _h: &H256) -> Option { + None + } + + fn import_block(&mut self, b: &[u8]) -> ImportResult { + let header = Rlp::new(&b).val_at::(0); + if header.number != From::from(self.blocks.len()) { + panic!("Unexpected block number"); + } + if !self.blocks.is_empty() { + let parent = Rlp::new(self.blocks.last().unwrap()).val_at::(0); + if header.parent_hash != parent.hash() { + panic!("Unexpected block header"); + } + } + self.difficulty = self.difficulty + header.difficulty; + self.last_hash = header.hash(); + self.hashes.insert(header.hash(), self.blocks.len()); + self.blocks.push(b.to_vec()); + ImportResult::Queued(QueueStatus::Known) + } + + fn queue_status(&self) -> BlockQueueStatus { + BlockQueueStatus { + full: false, + } + } + + fn clear_queue(&mut self) { + } + + fn info(&self) -> BlockChainInfo { + BlockChainInfo { + total_difficulty: self.difficulty, + pending_total_difficulty: self.difficulty, + genesis_hash: self.genesis_hash, + last_block_hash: self.last_hash, + last_block_number: self.blocks.len() as BlockNumber - 1, + } + } +} + +struct TestIo<'p> { + chain: &'p mut TestBlockChainClient, + queue: &'p mut VecDeque, + sender: Option, +} + +impl<'p> TestIo<'p> { + fn new(chain: &'p mut TestBlockChainClient, queue: &'p mut VecDeque, sender: Option) -> TestIo<'p> { + TestIo { + chain: chain, + queue: queue, + sender: sender + } + } +} + +impl<'p> SyncIo for TestIo<'p> { + fn disable_peer(&mut self, _peer_id: &PeerId) { + } + + fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError> { + self.queue.push_back(TestPacket { + data: data, + packet_id: packet_id, + recipient: self.sender.unwrap() + }); + Ok(()) + } + + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError> { + self.queue.push_back(TestPacket { + data: data, + packet_id: packet_id, + recipient: peer_id, + }); + Ok(()) + } + + fn chain<'a>(&'a mut self) -> &'a mut BlockChainClient { + self.chain + } +} + +struct TestPacket { + data: Bytes, + packet_id: PacketId, + recipient: PeerId, +} + +struct TestPeer { + chain: TestBlockChainClient, + sync: ChainSync, + queue: VecDeque, +} + +struct TestNet { + peers: Vec +} + +impl TestNet { + pub fn new(n: usize) -> TestNet { + let mut net = TestNet { + peers: Vec::new(), + }; + for _ in 0..n { + net.peers.push(TestPeer { + chain: TestBlockChainClient::new(), + sync: ChainSync::new(), + queue: VecDeque::new(), + }); + } + net + } + + pub fn peer(&mut self, i: usize) -> &mut TestPeer { + self.peers.get_mut(i).unwrap() + } + + pub fn start(&mut self) { + for peer in 0..self.peers.len() { + for client in 0..self.peers.len() { + if peer != client { + let mut p = self.peers.get_mut(peer).unwrap(); + p.sync.on_peer_connected(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(client as PeerId)), &(client as PeerId)); + } + } + } + } + + pub fn sync_step(&mut self) { + for peer in 0..self.peers.len() { + match self.peers[peer].queue.pop_front() { + Some(packet) => { + let mut p = self.peers.get_mut(packet.recipient).unwrap(); + p.sync.on_packet(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(peer as PeerId)), &(peer as PeerId), packet.packet_id, &packet.data); + }, + None => {} + } + let mut p = self.peers.get_mut(peer).unwrap(); + p.sync.maintain_sync(&mut TestIo::new(&mut p.chain, &mut p.queue, None)); + } + } + + pub fn sync(&mut self) { + self.start(); + while !self.done() { + self.sync_step() + } + } + + pub fn done(&self) -> bool { + self.peers.iter().all(|p| p.queue.is_empty()) + } } #[test] -fn full_sync() { +fn full_sync_two_peers() { + let mut net = TestNet::new(3); + net.peer(1).chain.add_blocks(1000, false); + net.peer(2).chain.add_blocks(1000, false); + net.sync(); + assert_eq!(net.peer(0).chain.block_at(50000), net.peer(1).chain.block_at(50000)); +} + +#[test] +fn full_sync_empty_blocks() { + let mut net = TestNet::new(3); + for n in 0..200 { + net.peer(1).chain.add_blocks(5, n % 2 == 0); + net.peer(2).chain.add_blocks(5, n % 2 == 0); + } + net.sync(); + assert_eq!(net.peer(0).chain.block_at(50000), net.peer(1).chain.block_at(50000)); } From 1d67a7a37343b8756a15539d1815ffd591b41fd0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 27 Dec 2015 00:48:03 +0100 Subject: [PATCH 05/27] sync fixed; more tests --- src/header.rs | 24 ++++---- src/sync/chain.rs | 108 ++++++++++++++++++++++++----------- src/sync/tests.rs | 140 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 185 insertions(+), 87 deletions(-) diff --git a/src/header.rs b/src/header.rs index fdc51a549..b9281144f 100644 --- a/src/header.rs +++ b/src/header.rs @@ -37,22 +37,22 @@ pub struct Header { impl Header { pub fn new() -> Header { Header { - parent_hash: ZERO_H256.clone(), - timestamp: BAD_U256.clone(), - number: ZERO_U256.clone(), - author: ZERO_ADDRESS.clone(), + parent_hash: ZERO_H256, + timestamp: BAD_U256, + number: ZERO_U256, + author: ZERO_ADDRESS, - transactions_root: ZERO_H256.clone(), - uncles_hash: ZERO_H256.clone(), + transactions_root: SHA3_NULL_RLP, + uncles_hash: SHA3_EMPTY_LIST_RLP, extra_data: vec![], - state_root: ZERO_H256.clone(), - receipts_root: ZERO_H256.clone(), - log_bloom: ZERO_LOGBLOOM.clone(), - gas_used: ZERO_U256.clone(), - gas_limit: ZERO_U256.clone(), + state_root: SHA3_NULL_RLP, + receipts_root: SHA3_NULL_RLP, + log_bloom: ZERO_LOGBLOOM, + gas_used: ZERO_U256, + gas_limit: ZERO_U256, - difficulty: ZERO_U256.clone(), + difficulty: ZERO_U256, seal: vec![], hash: Cell::new(None), } diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 0b921f552..d1d7df03b 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -2,7 +2,7 @@ use std::collections::{HashSet, HashMap}; use std::cmp::{min, max}; use std::mem::{replace}; use util::network::{PeerId, PacketId}; -use util::hash::{H256}; +use util::hash::{H256, FixedHash}; use util::bytes::{Bytes}; use util::uint::{U256}; use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp @@ -26,10 +26,10 @@ impl FromUsize for BlockNumber { const PROTOCOL_VERSION: u8 = 63u8; const MAX_BODIES_TO_SEND: usize = 256; -const MAX_HEADERS_TO_SEND: usize = 1024; +const MAX_HEADERS_TO_SEND: usize = 512; const MAX_NODE_DATA_TO_SEND: usize = 1024; const MAX_RECEIPTS_TO_SEND: usize = 1024; -const MAX_HEADERS_TO_REQUEST: usize = 1024; +const MAX_HEADERS_TO_REQUEST: usize = 512; const MAX_BODIES_TO_REQUEST: usize = 256; const STATUS_PACKET: u8 = 0x00; @@ -127,6 +127,8 @@ pub struct ChainSync { header_ids: HashMap, /// Last impoted block number last_imported_block: BlockNumber, + /// Last impoted block hash + last_imported_hash: H256, /// Syncing total difficulty syncing_difficulty: U256, /// True if common block for our and remote chain has been found @@ -147,6 +149,7 @@ impl ChainSync { peers: HashMap::new(), header_ids: HashMap::new(), last_imported_block: 0, + last_imported_hash: H256::new(), syncing_difficulty: U256::from(0u64), have_common_block: false } @@ -193,6 +196,7 @@ impl ChainSync { pub fn restart(&mut self, io: &mut SyncIo) { self.reset(); self.last_imported_block = 0; + self.last_imported_hash = H256::new(); self.starting_block = 0; self.highest_block = 0; self.have_common_block = false; @@ -224,8 +228,9 @@ impl ChainSync { /// Called by peer once it has new block headers during sync fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + self.reset_peer_asking(peer_id, PeerAsking::BlockHeaders); let item_count = r.item_count(); - trace!(target: "sync", "BlockHeaders ({} entries)", item_count); + trace!(target: "sync", "{}-> BlockHeaders ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block headers"); @@ -250,10 +255,16 @@ impl ChainSync { BlockStatus::InChain => { self.have_common_block = true; self.last_imported_block = number; + self.last_imported_hash = info.hash(); + trace!(target: "sync", "Found common header {} ({})", number, info.hash()); }, _ => { if self.have_common_block { //validate chain + if number == self.last_imported_block + 1 && info.parent_hash != self.last_imported_hash { + debug!(target: "sync", "Mismatched block header {} {}", number, info.hash()); + continue; + } if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash) { // mismatching parent id, delete the previous block and don't add this one // TODO: lower peer rating @@ -277,11 +288,12 @@ impl ChainSync { transactions_root: info.transactions_root, uncles: info.uncles_hash }; + trace!(target: "sync", "Got header {} ({})", number, info.hash()); if header_id.transactions_root == rlp::SHA3_NULL_RLP && header_id.uncles == rlp::SHA3_EMPTY_LIST_RLP { //empty body, just mark as downloaded let mut body_stream = RlpStream::new_list(2); - body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); body_stream.append_raw(&rlp::NULL_RLP, 1); + body_stream.append_raw(&rlp::EMPTY_LIST_RLP, 1); self.bodies.insert_item(number, body_stream.out()); } else { @@ -289,15 +301,16 @@ impl ChainSync { } } } - self.collect_blocks(io); - self.continue_sync(io); } + self.collect_blocks(io); + self.continue_sync(io); } /// Called by peer once it has new block bodies fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + self.reset_peer_asking(peer_id, PeerAsking::BlockBodies); let item_count = r.item_count(); - trace!(target: "sync", "BlockBodies ({} entries)", item_count); + trace!(target: "sync", "{}-> BlockBodies ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block bodies"); @@ -320,6 +333,7 @@ impl ChainSync { Some(n) => { self.header_ids.remove(&header_id); self.bodies.insert_item(n, body.raw().to_vec()); + trace!(target: "sync", "Got body {}", n); } None => { debug!(target: "sync", "Ignored unknown block body"); @@ -336,6 +350,7 @@ impl ChainSync { let header_rlp = block_rlp.at(0); let h = header_rlp.raw().sha3(); + trace!(target: "sync", "{}-> NewBlock ({})", peer_id, h); match io.chain().import_block(block_rlp.raw()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "New block already in chain {:?}", h); @@ -368,6 +383,7 @@ impl ChainSync { trace!(target: "sync", "Ignoring new hashes since we're already downloading."); return; } + trace!(target: "sync", "{}-> NewHashes ({} entries)", peer_id, r.item_count()); let hashes = r.iter().map(|item| (item.val_at::(0), item.val_at::(1))); let mut max_height: U256 = From::from(0); for (h, d) in hashes { @@ -397,11 +413,13 @@ impl ChainSync { /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: &PeerId) { + trace!(target: "sync", "== Disconnected {}", peer); self.clear_peer_download(peer); self.continue_sync(io); } pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: &PeerId) { + trace!(target: "sync", "== Connected {}", peer); self.send_status(io, peer); } @@ -415,6 +433,7 @@ impl ChainSync { /// Called after all blocks have been donloaded fn complete_sync(&mut self) { + trace!(target: "sync", "Sync complete"); self.reset(); self.state = SyncState::Idle; } @@ -428,13 +447,10 @@ impl ChainSync { fn sync_peer(&mut self, io: &mut SyncIo, peer_id: &PeerId, force: bool) { let (peer_latest, peer_difficulty) = { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); - if peer.asking != PeerAsking::Nothing - { - trace!(target: "sync", "Can't sync with this peer - outstanding asks."); + if peer.asking != PeerAsking::Nothing { return; } - if self.state == SyncState::Waiting - { + if self.state == SyncState::Waiting { trace!(target: "sync", "Waiting for block queue"); return; } @@ -449,6 +465,7 @@ impl ChainSync { if self.state == SyncState::Idle || self.state == SyncState::NotSynced { self.state = SyncState::Blocks; } + trace!(target: "sync", "Starting sync with better chain"); self.request_headers_by_hash(io, peer_id, &peer_latest, 1, 0, false); } else if self.state == SyncState::Blocks { @@ -470,7 +487,7 @@ impl ChainSync { if self.have_common_block && !self.headers.is_empty() && self.headers.range_iter().next().unwrap().0 == self.last_imported_block + 1 { for (start, ref items) in self.headers.range_iter() { - if needed_bodies.len() >= MAX_BODIES_TO_REQUEST { + if needed_bodies.len() > MAX_BODIES_TO_REQUEST { break; } let mut index: BlockNumber = 0; @@ -498,36 +515,34 @@ impl ChainSync { if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); } - if start <= 1 { + if start == 0 { self.have_common_block = true; //reached genesis } } if self.have_common_block { - - - let mut headers: Vec = Vec::new(); let mut prev = self.last_imported_block + 1; - for (start, ref items) in self.headers.range_iter() { - if headers.len() >= MAX_HEADERS_TO_REQUEST { + for (next, ref items) in self.headers.range_iter() { + if !headers.is_empty() { break; } - if start > prev { + if next <= prev { + prev = next + items.len() as BlockNumber; continue; } - let mut index = 0; - while index != items.len() as BlockNumber && headers.len() < MAX_BODIES_TO_REQUEST { - let block = prev + index; + let mut block = prev; + while block < next && headers.len() <= MAX_HEADERS_TO_REQUEST { if !self.downloading_headers.contains(&(block as BlockNumber)) { headers.push(block as BlockNumber); self.downloading_headers.insert(block as BlockNumber); } - index += 1; + block += 1; } - prev = start + items.len() as BlockNumber; + prev = next + items.len() as BlockNumber; } if !headers.is_empty() { + start = headers[0] as usize; let count = headers.len(); replace(&mut self.peers.get_mut(peer_id).unwrap().asking_blocks, headers); assert!(!self.headers.have_item(&(start as BlockNumber))); @@ -563,23 +578,31 @@ impl ChainSync { return; } - for i in 0..min(headers.1.len(), bodies.1.len()) { + let count = min(headers.1.len(), bodies.1.len()); + let mut imported = 0; + for i in 0..count { let mut block_rlp = RlpStream::new_list(3); block_rlp.append_raw(&headers.1[i].data, 1); - block_rlp.append_raw(&bodies.1[i], 2); + let body = Rlp::new(&bodies.1[i]); + block_rlp.append_raw(body.at(0).raw(), 1); + block_rlp.append_raw(body.at(1).raw(), 1); let h = &headers.1[i].hash; match io.chain().import_block(&block_rlp.out()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; + self.last_imported_hash = *h; }, ImportResult::AlreadyQueued(_) => { trace!(target: "sync", "Block already queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; + self.last_imported_hash = *h; }, ImportResult::Queued(QueueStatus::Known) => { trace!(target: "sync", "Block queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; + self.last_imported_hash = *h; + imported += 1; }, ImportResult::Queued(QueueStatus::Unknown) => { panic!("Queued out of order block"); @@ -590,6 +613,7 @@ impl ChainSync { } } } + trace!(target: "sync", "Imported {} of {}", imported, count); } if restart { @@ -627,6 +651,7 @@ impl ChainSync { } fn request_headers_by_hash(&mut self, sync: &mut SyncIo, peer_id: &PeerId, h: &H256, count: usize, skip: usize, reverse: bool) { + trace!(target: "sync", "{}<- GetBlockHeaders: {} entries starting from {}", peer_id, count, h); let mut rlp = RlpStream::new_list(4); rlp.append(h); rlp.append(&count); @@ -637,6 +662,7 @@ impl ChainSync { fn request_headers_by_number(&mut self, sync: &mut SyncIo, peer_id: &PeerId, n: BlockNumber, count: usize, skip: usize, reverse: bool) { let mut rlp = RlpStream::new_list(4); + trace!(target: "sync", "{}<- GetBlockHeaders: {} entries starting from {}", peer_id, count, n); rlp.append(&n); rlp.append(&count); rlp.append(&skip); @@ -646,12 +672,23 @@ impl ChainSync { fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: &PeerId, hashes: Vec) { let mut rlp = RlpStream::new_list(hashes.len()); + trace!(target: "sync", "{}<- GetBlockBodies: {} entries", peer_id, hashes.len()); for h in hashes { rlp.append(&h); } self.send_request(sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); } + fn reset_peer_asking(&mut self, peer_id: &PeerId, asking: PeerAsking) { + let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); + if peer.asking != asking { + warn!(target:"sync", "Asking {:?} while expected {:?}", peer.asking, asking); + } + else { + peer.asking = PeerAsking::Nothing; + } + } + fn send_request(&mut self, sync: &mut SyncIo, peer_id: &PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); @@ -703,22 +740,26 @@ impl ChainSync { let mut number = if r.at(0).size() == 32 { // id is a hash let hash: H256 = r.val_at(0); - trace!(target: "sync", "GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); + trace!(target: "sync", "-> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); match io.chain().block_header(&hash) { Some(hdr) => From::from(rlp::decode::(&hdr).number), None => last } } else { + trace!(target: "sync", "-> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", r.val_at::(0), max_headers, skip, reverse); r.val_at(0) }; - number = max(1, number); - number = min(last, number); + if reverse { + number = min(last, number); + } else { + number = max(1, number); + } let max_count = min(MAX_HEADERS_TO_SEND, max_headers); let mut count = 0; let mut data = Bytes::new(); - while number < last && number > 1 && count < max_count { + while number <= last && number > 0 && count < max_count { match io.chain().block_header_at(number) { Some(mut hdr) => { data.append(&mut hdr); @@ -737,6 +778,7 @@ impl ChainSync { rlp.append_raw(&data, count as usize); io.respond(BLOCK_HEADERS_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); + trace!(target: "sync", "-> GetBlockHeaders: returned {} entries", count); } fn return_block_bodies(&self, io: &mut SyncIo, r: &Rlp) { @@ -745,6 +787,7 @@ impl ChainSync { debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); return; } + trace!(target: "sync", "-> GetBlockBodies: {} entries", count); count = min(count, MAX_BODIES_TO_SEND); let mut added = 0usize; let mut data = Bytes::new(); @@ -761,6 +804,7 @@ impl ChainSync { rlp.append_raw(&data, added); io.respond(BLOCK_BODIES_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); + trace!(target: "sync", "-> GetBlockBodies: returned {} entries", added); } fn return_node_data(&self, io: &mut SyncIo, r: &Rlp) { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 7be66ff2f..6b2db9c9b 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -10,8 +10,8 @@ use sync::{SyncIo}; use sync::chain::{ChainSync}; struct TestBlockChainClient { - blocks: Vec, - hashes: HashMap, + blocks: HashMap, + numbers: HashMap, genesis_hash: H256, last_hash: H256, difficulty: U256 @@ -21,8 +21,8 @@ impl TestBlockChainClient { fn new() -> TestBlockChainClient { let mut client = TestBlockChainClient { - blocks: Vec::new(), - hashes: HashMap::new(), + blocks: HashMap::new(), + numbers: HashMap::new(), genesis_hash: H256::new(), last_hash: H256::new(), difficulty: From::from(0), @@ -33,20 +33,20 @@ impl TestBlockChainClient { } pub fn add_blocks(&mut self, count: usize, empty: bool) { - for n in self.blocks.len()..(self.blocks.len() + count) { + for n in self.numbers.len()..(self.numbers.len() + count) { let mut header = BlockHeader::new(); header.difficulty = From::from(n); header.parent_hash = self.last_hash; header.number = From::from(n); let mut uncles = RlpStream::new_list(if empty {0} else {1}); if !empty { - uncles.append(&H256::random()); + uncles.append(&H256::from(&U256::from(n))); header.uncles_hash = uncles.raw().sha3(); } let mut rlp = RlpStream::new_list(3); rlp.append(&header); - rlp.append_raw(uncles.raw(), 1); rlp.append_raw(&rlp::NULL_RLP, 1); + rlp.append_raw(uncles.raw(), 1); self.import_block(rlp.raw()); } } @@ -54,27 +54,12 @@ impl TestBlockChainClient { impl BlockChainClient for TestBlockChainClient { fn block_header(&self, h: &H256) -> Option { - self.hashes.get(h).and_then(|i| self.block_header_at(*i as BlockNumber)) + self.blocks.get(h).map(|r| Rlp::new(r).at(0).raw().to_vec()) + } fn block_body(&self, h: &H256) -> Option { - self.hashes.get(h).and_then(|i| self.block_body_at(*i as BlockNumber)) - } - - fn block(&self, h: &H256) -> Option { - self.hashes.get(h).map(|i| self.blocks[*i].clone()) - } - - fn block_status(&self, h: &H256) -> BlockStatus { - self.hashes.get(h).map(|i| self.block_status_at(*i as BlockNumber)).unwrap_or(BlockStatus::Unknown) - } - - fn block_header_at(&self, n: BlockNumber) -> Option { - self.blocks.get(n as usize).map(|r| Rlp::new(r).at(0).raw().to_vec()) - } - - fn block_body_at(&self, n: BlockNumber) -> Option { - self.blocks.get(n as usize).map(|r| { + self.blocks.get(h).map(|r| { let mut stream = RlpStream::new_list(2); stream.append_raw(Rlp::new(&r).at(1).raw(), 1); stream.append_raw(Rlp::new(&r).at(2).raw(), 1); @@ -82,8 +67,27 @@ impl BlockChainClient for TestBlockChainClient { }) } + fn block(&self, h: &H256) -> Option { + self.blocks.get(h).map(|b| b.clone()) + } + + fn block_status(&self, h: &H256) -> BlockStatus { + match self.blocks.get(h) { + Some(_) => BlockStatus::InChain, + None => BlockStatus::Unknown + } + } + + fn block_header_at(&self, n: BlockNumber) -> Option { + self.numbers.get(&(n as usize)).and_then(|h| self.block_header(h)) + } + + fn block_body_at(&self, n: BlockNumber) -> Option { + self.numbers.get(&(n as usize)).and_then(|h| self.block_body(h)) + } + fn block_at(&self, n: BlockNumber) -> Option { - self.blocks.get(n as usize).map(|b| b.clone()) + self.numbers.get(&(n as usize)).map(|h| self.blocks.get(h).unwrap().clone()) } fn block_status_at(&self, n: BlockNumber) -> BlockStatus { @@ -112,19 +116,41 @@ impl BlockChainClient for TestBlockChainClient { fn import_block(&mut self, b: &[u8]) -> ImportResult { let header = Rlp::new(&b).val_at::(0); - if header.number != From::from(self.blocks.len()) { - panic!("Unexpected block number"); + let number: usize = header.number.low_u64() as usize; + if number > self.blocks.len() { + panic!("Unexpected block number. Expected {}, got {}", self.blocks.len(), number); } - if !self.blocks.is_empty() { - let parent = Rlp::new(self.blocks.last().unwrap()).val_at::(0); - if header.parent_hash != parent.hash() { - panic!("Unexpected block header"); + if number > 0 { + match self.blocks.get(&header.parent_hash) { + Some(parent) => { + let parent = Rlp::new(parent).val_at::(0); + if parent.number != (header.number - From::from(1)) { + panic!("Unexpected block parent"); + } + }, + None => { + panic!("Unknown block parent {:?} for block {}", header.parent_hash, number); + } } } - self.difficulty = self.difficulty + header.difficulty; - self.last_hash = header.hash(); - self.hashes.insert(header.hash(), self.blocks.len()); - self.blocks.push(b.to_vec()); + if number == self.numbers.len() { + self.difficulty = self.difficulty + header.difficulty; + self.last_hash = header.hash(); + self.blocks.insert(header.hash(), b.to_vec()); + self.numbers.insert(number, header.hash()); + let mut parent_hash = header.parent_hash; + if number > 0 { + let mut n = number - 1; + while n > 0 && self.numbers[&n] != parent_hash { + *self.numbers.get_mut(&n).unwrap() = parent_hash; + n -= 1; + parent_hash = Rlp::new(&self.blocks[&parent_hash]).val_at::(0).parent_hash; + } + } + } + else { + self.blocks.insert(header.hash(), b.to_vec()); + } ImportResult::Queued(QueueStatus::Known) } @@ -222,7 +248,11 @@ impl TestNet { net } - pub fn peer(&mut self, i: usize) -> &mut TestPeer { + pub fn peer(&self, i: usize) -> &TestPeer { + self.peers.get(i).unwrap() + } + + pub fn peer_mut(&mut self, i: usize) -> &mut TestPeer { self.peers.get_mut(i).unwrap() } @@ -242,7 +272,9 @@ impl TestNet { match self.peers[peer].queue.pop_front() { Some(packet) => { let mut p = self.peers.get_mut(packet.recipient).unwrap(); + trace!("--- {} -> {} ---", peer, packet.recipient); p.sync.on_packet(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(peer as PeerId)), &(peer as PeerId), packet.packet_id, &packet.data); + trace!("----------------"); }, None => {} } @@ -267,19 +299,41 @@ impl TestNet { #[test] fn full_sync_two_peers() { let mut net = TestNet::new(3); - net.peer(1).chain.add_blocks(1000, false); - net.peer(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, false); + net.peer_mut(2).chain.add_blocks(1000, false); net.sync(); - assert_eq!(net.peer(0).chain.block_at(50000), net.peer(1).chain.block_at(50000)); + assert!(net.peer(0).chain.block_at(1000).is_some()); + assert_eq!(net.peer(0).chain.blocks, net.peer(1).chain.blocks); } #[test] fn full_sync_empty_blocks() { let mut net = TestNet::new(3); for n in 0..200 { - net.peer(1).chain.add_blocks(5, n % 2 == 0); - net.peer(2).chain.add_blocks(5, n % 2 == 0); + net.peer_mut(1).chain.add_blocks(5, n % 2 == 0); + net.peer_mut(2).chain.add_blocks(5, n % 2 == 0); } net.sync(); - assert_eq!(net.peer(0).chain.block_at(50000), net.peer(1).chain.block_at(50000)); + assert!(net.peer(0).chain.block_at(1000).is_some()); + assert_eq!(net.peer(0).chain.blocks, net.peer(1).chain.blocks); +} + +#[test] +fn forked_sync() { + ::env_logger::init().ok(); + let mut net = TestNet::new(3); + net.peer_mut(0).chain.add_blocks(300, false); + net.peer_mut(1).chain.add_blocks(300, false); + net.peer_mut(2).chain.add_blocks(300, false); + net.peer_mut(0).chain.add_blocks(100, true); //fork + net.peer_mut(1).chain.add_blocks(200, false); + net.peer_mut(2).chain.add_blocks(200, false); + net.peer_mut(1).chain.add_blocks(100, false); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, true); + // peer 1 has the best chain of 601 blocks + let peer1_chain = net.peer(1).chain.numbers.clone(); + net.sync(); + assert_eq!(net.peer(0).chain.numbers, peer1_chain); + assert_eq!(net.peer(1).chain.numbers, peer1_chain); + assert_eq!(net.peer(2).chain.numbers, peer1_chain); } From 1a1c61179f8026fb7042d151ae8c6ebf122aa5fb Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 27 Dec 2015 02:27:15 +0100 Subject: [PATCH 06/27] order peers by difficulty --- src/sync/chain.rs | 14 +++++++++----- src/sync/tests.rs | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index d1d7df03b..41979293d 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -261,7 +261,8 @@ impl ChainSync { _ => { if self.have_common_block { //validate chain - if number == self.last_imported_block + 1 && info.parent_hash != self.last_imported_hash { + if self.have_common_block && number == self.last_imported_block + 1 && info.parent_hash != self.last_imported_hash { + // TODO: lower peer rating debug!(target: "sync", "Mismatched block header {} {}", number, info.hash()); continue; } @@ -423,10 +424,11 @@ impl ChainSync { self.send_status(io, peer); } - /// Resume downloading after witing state + /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { - let peers: Vec = self.peers.keys().map(|k| *k).collect(); - for p in peers { + let mut peers: Vec<(PeerId, U256)> = self.peers.iter().map(|(k, p)| (*k, p.difficulty)).collect(); + peers.sort_by(|&(_, d1), &(_, d2)| d1.cmp(&d2).reverse()); //TODO: sort by rating + for (p, _) in peers { self.sync_peer(io, &p, false); } } @@ -511,12 +513,14 @@ impl ChainSync { let mut start = 0usize; if !self.have_common_block { // download backwards until common block is found 1 header at a time - start = io.chain().info().last_block_number as usize; + let chain_info = io.chain().info(); + start = chain_info.last_block_number as usize; if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); } if start == 0 { self.have_common_block = true; //reached genesis + self.last_imported_hash = chain_info.genesis_hash; } } if self.have_common_block { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 6b2db9c9b..21ba006a6 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -298,6 +298,7 @@ impl TestNet { #[test] fn full_sync_two_peers() { + ::env_logger::init().ok(); let mut net = TestNet::new(3); net.peer_mut(1).chain.add_blocks(1000, false); net.peer_mut(2).chain.add_blocks(1000, false); @@ -308,6 +309,7 @@ fn full_sync_two_peers() { #[test] fn full_sync_empty_blocks() { + ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { net.peer_mut(1).chain.add_blocks(5, n % 2 == 0); From b925df2cd98a57b29c2d3f5f1e216379d0ed1c9c Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 28 Dec 2015 12:03:05 +0100 Subject: [PATCH 07/27] Use HeaderView instead of BlockHeader --- src/eth.rs | 1 - src/sync/chain.rs | 42 ++++++++++++++++++++++-------------------- src/sync/mod.rs | 5 ++++- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/eth.rs b/src/eth.rs index 5eb7e9b6c..4ddcaa3f6 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -38,7 +38,6 @@ pub struct BlockQueueStatus { pub type TreeRoute = ::blockchain::TreeRoute; pub type BlockNumber = u32; -pub type BlockHeader = ::header::Header; pub trait BlockChainClient : Sync { fn block_header(&self, h: &H256) -> Option; diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 41979293d..0954e6760 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -8,7 +8,8 @@ use util::uint::{U256}; use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp use util::rlp::rlptraits::{Stream, View}; use util::sha3::Hashable; -use eth::{BlockNumber, BlockChainClient, BlockHeader, BlockStatus, QueueStatus, ImportResult}; +use eth::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; +use views::{HeaderView}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; use sync::{SyncIo}; @@ -242,8 +243,8 @@ impl ChainSync { } for i in 0..item_count { - let info: BlockHeader = r.val_at(i); - let number = BlockNumber::from(info.number); + let info = HeaderView::new_from_rlp(r.at(i)); + let number = BlockNumber::from(info.number()); if number <= self.last_imported_block || self.headers.have_item(&number) { trace!(target: "sync", "Skipping existing block header"); continue; @@ -251,29 +252,30 @@ impl ChainSync { if number > self.highest_block { self.highest_block = number; } - match io.chain().block_status(&info.hash()) { + let hash = info.sha3(); + match io.chain().block_status(&hash) { BlockStatus::InChain => { self.have_common_block = true; self.last_imported_block = number; - self.last_imported_hash = info.hash(); - trace!(target: "sync", "Found common header {} ({})", number, info.hash()); + self.last_imported_hash = hash; + trace!(target: "sync", "Found common header {} ({})", number, hash); }, _ => { if self.have_common_block { //validate chain - if self.have_common_block && number == self.last_imported_block + 1 && info.parent_hash != self.last_imported_hash { + if self.have_common_block && number == self.last_imported_block + 1 && info.parent_hash() != self.last_imported_hash { // TODO: lower peer rating - debug!(target: "sync", "Mismatched block header {} {}", number, info.hash()); + debug!(target: "sync", "Mismatched block header {} {}", number, hash); continue; } - if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash) { + if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash()) { // mismatching parent id, delete the previous block and don't add this one // TODO: lower peer rating - debug!(target: "sync", "Mismatched block header {} {}", number, info.hash()); + debug!(target: "sync", "Mismatched block header {} {}", number, hash); self.remove_downloaded_blocks(number - 1); continue; } - if self.headers.find_item(&(number + 1)).map_or(false, |p| p.parent != info.hash()) { + if self.headers.find_item(&(number + 1)).map_or(false, |p| p.parent != hash) { // mismatching parent id for the next block, clear following headers debug!(target: "sync", "Mismatched block header {}", number + 1); self.remove_downloaded_blocks(number + 1); @@ -281,15 +283,15 @@ impl ChainSync { } let hdr = Header { data: r.at(i).raw().to_vec(), - hash: info.hash(), - parent: info.parent_hash, + hash: hash, + parent: info.parent_hash(), }; self.headers.insert_item(number, hdr); let header_id = HeaderId { - transactions_root: info.transactions_root, - uncles: info.uncles_hash + transactions_root: info.transactions_root(), + uncles: info.uncles_hash() }; - trace!(target: "sync", "Got header {} ({})", number, info.hash()); + trace!(target: "sync", "Got header {} ({})", number, hash); if header_id.transactions_root == rlp::SHA3_NULL_RLP && header_id.uncles == rlp::SHA3_EMPTY_LIST_RLP { //empty body, just mark as downloaded let mut body_stream = RlpStream::new_list(2); @@ -638,10 +640,10 @@ impl ChainSync { for n in self.headers.get_tail(&start) { match self.headers.find_item(&n) { Some(ref header_data) => { - let header_to_delete: BlockHeader = rlp::decode(&header_data.data); + let header_to_delete = HeaderView::new(&header_data.data); let header_id = HeaderId { - transactions_root: header_to_delete.transactions_root, - uncles: header_to_delete.uncles_hash + transactions_root: header_to_delete.transactions_root(), + uncles: header_to_delete.uncles_hash() }; self.header_ids.remove(&header_id); }, @@ -746,7 +748,7 @@ impl ChainSync { let hash: H256 = r.val_at(0); trace!(target: "sync", "-> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); match io.chain().block_header(&hash) { - Some(hdr) => From::from(rlp::decode::(&hdr).number), + Some(hdr) => From::from(HeaderView::new(&hdr).number()), None => last } } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 8efb86a43..03a03f49d 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,6 +1,6 @@ use std::sync::{Arc}; use eth::{BlockChainClient}; -use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, PacketId, Error as NetworkError}; +use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, PacketId, Message, Error as NetworkError}; use sync::chain::{ChainSync}; mod chain; @@ -101,6 +101,9 @@ impl ProtocolHandler for EthSync { fn timeout(&mut self, io: &mut HandlerIo, _timer: TimerToken) { self.sync.maintain_sync(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } + + fn message(&mut self, _io: &mut HandlerIo, _message: &Message) { + } } From 8d37ef7d8e9c1a2f7bcb2466e46520131fe8c688 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 4 Jan 2016 13:25:32 +0100 Subject: [PATCH 08/27] Removed Copy trait from H256 --- src/genesis.rs | 4 ++-- src/header.rs | 26 +++++++++++++------------- src/sync/chain.rs | 10 +++++----- src/sync/tests.rs | 13 +++++++------ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/genesis.rs b/src/genesis.rs index 3e95c098a..13d228d79 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,6 +1,6 @@ use std::io::Read; use std::str::FromStr; -use std::cell::Cell; +use std::cell::RefCell; use std::collections::HashMap; use rustc_serialize::base64::FromBase64; use rustc_serialize::json::Json; @@ -83,7 +83,7 @@ impl Genesis { let nonce = H64::from_str(&json["nonce"].as_string().unwrap()[2..]).unwrap(); vec![mixhash.to_vec(), nonce.to_vec()] }, - hash: Cell::new(None) + hash: RefCell::new(None) }; let mut state = HashMap::new(); diff --git a/src/header.rs b/src/header.rs index b9281144f..2a581f6ce 100644 --- a/src/header.rs +++ b/src/header.rs @@ -1,4 +1,4 @@ -use std::cell::Cell; +use std::cell::RefCell; use util::hash::*; use util::sha3::*; use util::bytes::*; @@ -31,16 +31,16 @@ pub struct Header { pub difficulty: U256, pub seal: Vec, - pub hash: Cell>, //TODO: make this private + pub hash: RefCell>, //TODO: make this private } impl Header { pub fn new() -> Header { Header { - parent_hash: ZERO_H256, + parent_hash: ZERO_H256.clone(), timestamp: BAD_U256, number: ZERO_U256, - author: ZERO_ADDRESS, + author: ZERO_ADDRESS.clone(), transactions_root: SHA3_NULL_RLP, uncles_hash: SHA3_EMPTY_LIST_RLP, @@ -48,26 +48,26 @@ impl Header { state_root: SHA3_NULL_RLP, receipts_root: SHA3_NULL_RLP, - log_bloom: ZERO_LOGBLOOM, + log_bloom: ZERO_LOGBLOOM.clone(), gas_used: ZERO_U256, gas_limit: ZERO_U256, difficulty: ZERO_U256, seal: vec![], - hash: Cell::new(None), + hash: RefCell::new(None), } } pub fn hash(&self) -> H256 { - let hash = self.hash.get(); - match hash { - Some(h) => h, - None => { + let mut hash = self.hash.borrow_mut(); + match &mut *hash { + &mut Some(ref h) => h.clone(), + hash @ &mut None => { let mut stream = RlpStream::new(); stream.append(self); let h = stream.raw().sha3(); - self.hash.set(Some(h.clone())); - h + *hash = Some(h.clone()); + h.clone() } } } @@ -92,7 +92,7 @@ impl Decodable for Header { timestamp: try!(r.val_at(11)), extra_data: try!(r.val_at(12)), seal: vec![], - hash: Cell::new(Some(r.raw().sha3())) + hash: RefCell::new(Some(r.raw().sha3())) }; for i in 13..r.item_count() { diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 0954e6760..5d89115e3 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -257,7 +257,7 @@ impl ChainSync { BlockStatus::InChain => { self.have_common_block = true; self.last_imported_block = number; - self.last_imported_hash = hash; + self.last_imported_hash = hash.clone(); trace!(target: "sync", "Found common header {} ({})", number, hash); }, _ => { @@ -283,7 +283,7 @@ impl ChainSync { } let hdr = Header { data: r.at(i).raw().to_vec(), - hash: hash, + hash: hash.clone(), parent: info.parent_hash(), }; self.headers.insert_item(number, hdr); @@ -597,17 +597,17 @@ impl ChainSync { ImportResult::AlreadyInChain => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; - self.last_imported_hash = *h; + self.last_imported_hash = h.clone(); }, ImportResult::AlreadyQueued(_) => { trace!(target: "sync", "Block already queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; - self.last_imported_hash = *h; + self.last_imported_hash = h.clone(); }, ImportResult::Queued(QueueStatus::Known) => { trace!(target: "sync", "Block queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; - self.last_imported_hash = *h; + self.last_imported_hash = h.clone(); imported += 1; }, ImportResult::Queued(QueueStatus::Unknown) => { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 21ba006a6..e5c8dfe43 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -5,7 +5,8 @@ use util::uint::{U256}; use util::sha3::Hashable; use util::rlp::{self, Rlp, RlpStream, View, Stream}; use util::network::{PeerId, PacketId, Error as NetworkError}; -use eth::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, BlockHeader, QueueStatus}; +use eth::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; +use header::Header as BlockHeader; use sync::{SyncIo}; use sync::chain::{ChainSync}; @@ -28,7 +29,7 @@ impl TestBlockChainClient { difficulty: From::from(0), }; client.add_blocks(1, true); // add genesis block - client.genesis_hash = client.last_hash; + client.genesis_hash = client.last_hash.clone(); client } @@ -36,7 +37,7 @@ impl TestBlockChainClient { for n in self.numbers.len()..(self.numbers.len() + count) { let mut header = BlockHeader::new(); header.difficulty = From::from(n); - header.parent_hash = self.last_hash; + header.parent_hash = self.last_hash.clone(); header.number = From::from(n); let mut uncles = RlpStream::new_list(if empty {0} else {1}); if !empty { @@ -142,7 +143,7 @@ impl BlockChainClient for TestBlockChainClient { if number > 0 { let mut n = number - 1; while n > 0 && self.numbers[&n] != parent_hash { - *self.numbers.get_mut(&n).unwrap() = parent_hash; + *self.numbers.get_mut(&n).unwrap() = parent_hash.clone(); n -= 1; parent_hash = Rlp::new(&self.blocks[&parent_hash]).val_at::(0).parent_hash; } @@ -167,8 +168,8 @@ impl BlockChainClient for TestBlockChainClient { BlockChainInfo { total_difficulty: self.difficulty, pending_total_difficulty: self.difficulty, - genesis_hash: self.genesis_hash, - last_block_hash: self.last_hash, + genesis_hash: self.genesis_hash.clone(), + last_block_hash: self.last_hash.clone(), last_block_number: self.blocks.len() as BlockNumber - 1, } } From 2a4d47003997aed7146c2cbf7b78703c570e4175 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 16:08:12 +0100 Subject: [PATCH 09/27] Client app --- src/bin/client.rs | 19 +++++++-- src/blockchain.rs | 81 ++++++++++++++++++------------------ src/eth.rs | 20 ++++----- src/lib.rs | 2 + src/sync/chain.rs | 8 ++-- src/sync/mod.rs | 7 ++++ src/sync/range_collection.rs | 2 +- src/sync/tests.rs | 4 +- 8 files changed, 83 insertions(+), 60 deletions(-) diff --git a/src/bin/client.rs b/src/bin/client.rs index 0af81e47f..e25248673 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -1,15 +1,28 @@ extern crate ethcore_util as util; +extern crate ethcore; +extern crate rustc_serialize; use std::io::*; +use std::env; +use std::sync::Arc; +use rustc_serialize::hex::FromHex; +use util::hash::*; use util::network::{NetworkService}; - +use ethcore::client::Client; +use ethcore::sync::EthSync; fn main() { - let _service = NetworkService::start().unwrap(); + let mut service = NetworkService::start().unwrap(); + let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + let client = Arc::new(Client::new(&genesis, &dir)); + let sync = Box::new(EthSync::new(client)); + service.register_protocol(sync, "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); loop { let mut cmd = String::new(); stdin().read_line(&mut cmd).unwrap(); - if cmd == "quit" || cmd == "exit" || cmd == "q" { + if cmd == "quit\n" || cmd == "exit\n" || cmd == "q\n" { break; } } diff --git a/src/blockchain.rs b/src/blockchain.rs index f64d81189..8eee914db 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -1,9 +1,9 @@ //! Fast access to blockchain data. use std::collections::HashMap; -use std::cell::RefCell; use std::path::Path; use std::hash::Hash; +use std::sync::RwLock; use rocksdb::{DB, WriteBatch, Writable}; use heapsize::HeapSizeOf; use util::hash::*; @@ -62,17 +62,17 @@ impl BestBlock { /// /// **Does not do input data verification.** pub struct BlockChain { - best_block: RefCell, + best_block: RwLock, // block cache - blocks: RefCell>, + blocks: RwLock>, // extra caches - block_details: RefCell>, - block_hashes: RefCell>, - transaction_addresses: RefCell>, - block_logs: RefCell>, - blocks_blooms: RefCell>, + block_details: RwLock>, + block_hashes: RwLock>, + transaction_addresses: RwLock>, + block_logs: RwLock>, + blocks_blooms: RwLock>, extras_db: DB, blocks_db: DB @@ -117,13 +117,13 @@ impl BlockChain { let blocks_db = DB::open_default(blocks_path.to_str().unwrap()).unwrap(); let bc = BlockChain { - best_block: RefCell::new(BestBlock::new()), - blocks: RefCell::new(HashMap::new()), - block_details: RefCell::new(HashMap::new()), - block_hashes: RefCell::new(HashMap::new()), - transaction_addresses: RefCell::new(HashMap::new()), - block_logs: RefCell::new(HashMap::new()), - blocks_blooms: RefCell::new(HashMap::new()), + best_block: RwLock::new(BestBlock::new()), + blocks: RwLock::new(HashMap::new()), + block_details: RwLock::new(HashMap::new()), + block_hashes: RwLock::new(HashMap::new()), + transaction_addresses: RwLock::new(HashMap::new()), + block_logs: RwLock::new(HashMap::new()), + blocks_blooms: RwLock::new(HashMap::new()), extras_db: extras_db, blocks_db: blocks_db }; @@ -158,7 +158,7 @@ impl BlockChain { }; { - let mut best_block = bc.best_block.borrow_mut(); + let mut best_block = bc.best_block.write().unwrap(); best_block.number = bc.block_number(&best_block_hash).unwrap(); best_block.total_difficulty = bc.block_details(&best_block_hash).unwrap().total_difficulty; best_block.hash = best_block_hash; @@ -272,7 +272,7 @@ impl BlockChain { // create views onto rlp let block = BlockView::new(bytes); let header = block.header_view(); - let hash = block.sha3(); + let hash = header.sha3(); if self.is_known(&hash) { return; @@ -283,13 +283,13 @@ impl BlockChain { let (batch, new_best) = self.block_to_extras_insert_batch(bytes); // update best block - let mut best_block = self.best_block.borrow_mut(); + let mut best_block = self.best_block.write().unwrap(); if let Some(b) = new_best { *best_block = b; } // update caches - let mut write = self.block_details.borrow_mut(); + let mut write = self.block_details.write().unwrap(); write.remove(&header.parent_hash()); // update extras database @@ -425,17 +425,17 @@ impl BlockChain { /// Get best block hash. pub fn best_block_hash(&self) -> H256 { - self.best_block.borrow().hash.clone() + self.best_block.read().unwrap().hash.clone() } /// Get best block number. pub fn best_block_number(&self) -> U256 { - self.best_block.borrow().number + self.best_block.read().unwrap().number } /// Get best block total difficulty. pub fn best_block_total_difficulty(&self) -> U256 { - self.best_block.borrow().total_difficulty + self.best_block.read().unwrap().total_difficulty } /// Get the number of given block's hash. @@ -448,9 +448,10 @@ impl BlockChain { self.query_extras(hash, &self.block_logs) } - fn block(&self, hash: &H256) -> Option { + /// Get raw block data + pub fn block(&self, hash: &H256) -> Option { { - let read = self.blocks.borrow(); + let read = self.blocks.read().unwrap(); match read.get(hash) { Some(v) => return Some(v.clone()), None => () @@ -463,7 +464,7 @@ impl BlockChain { match opt { Some(b) => { let bytes: Bytes = b.to_vec(); - let mut write = self.blocks.borrow_mut(); + let mut write = self.blocks.write().unwrap(); write.insert(hash.clone(), bytes.clone()); Some(bytes) }, @@ -471,11 +472,11 @@ impl BlockChain { } } - fn query_extras(&self, hash: &K, cache: &RefCell>) -> Option where + fn query_extras(&self, hash: &K, cache: &RwLock>) -> Option where T: Clone + Decodable + ExtrasIndexable, K: ExtrasSliceConvertable + Eq + Hash + Clone { { - let read = cache.borrow(); + let read = cache.read().unwrap(); match read.get(hash) { Some(v) => return Some(v.clone()), None => () @@ -483,17 +484,17 @@ impl BlockChain { } self.extras_db.get_extras(hash).map(| t: T | { - let mut write = cache.borrow_mut(); + let mut write = cache.write().unwrap(); write.insert(hash.clone(), t.clone()); t }) } - fn query_extras_exist(&self, hash: &K, cache: &RefCell>) -> bool where + fn query_extras_exist(&self, hash: &K, cache: &RwLock>) -> bool where K: ExtrasSliceConvertable + Eq + Hash + Clone, T: ExtrasIndexable { { - let read = cache.borrow(); + let read = cache.read().unwrap(); match read.get(hash) { Some(_) => return true, None => () @@ -506,21 +507,21 @@ impl BlockChain { /// Get current cache size. pub fn cache_size(&self) -> CacheSize { CacheSize { - blocks: self.blocks.heap_size_of_children(), - block_details: self.block_details.heap_size_of_children(), - transaction_addresses: self.transaction_addresses.heap_size_of_children(), - block_logs: self.block_logs.heap_size_of_children(), - blocks_blooms: self.blocks_blooms.heap_size_of_children() + blocks: self.blocks.read().unwrap().heap_size_of_children(), + block_details: self.block_details.read().unwrap().heap_size_of_children(), + transaction_addresses: self.transaction_addresses.read().unwrap().heap_size_of_children(), + block_logs: self.block_logs.read().unwrap().heap_size_of_children(), + blocks_blooms: self.blocks_blooms.read().unwrap().heap_size_of_children() } } /// Tries to squeeze the cache if its too big. pub fn squeeze_to_fit(&self, size: CacheSize) { - self.blocks.borrow_mut().squeeze(size.blocks); - self.block_details.borrow_mut().squeeze(size.block_details); - self.transaction_addresses.borrow_mut().squeeze(size.transaction_addresses); - self.block_logs.borrow_mut().squeeze(size.block_logs); - self.blocks_blooms.borrow_mut().squeeze(size.blocks_blooms); + self.blocks.write().unwrap().squeeze(size.blocks); + self.block_details.write().unwrap().squeeze(size.block_details); + self.transaction_addresses.write().unwrap().squeeze(size.transaction_addresses); + self.block_logs.write().unwrap().squeeze(size.block_logs); + self.blocks_blooms.write().unwrap().squeeze(size.blocks_blooms); } } diff --git a/src/eth.rs b/src/eth.rs index 4ddcaa3f6..97183d983 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -27,8 +27,8 @@ pub struct BlockChainInfo { pub total_difficulty: U256, pub pending_total_difficulty: U256, pub genesis_hash: H256, - pub last_block_hash: H256, - pub last_block_number: BlockNumber + pub best_block_hash: H256, + pub best_block_number: BlockNumber } pub struct BlockQueueStatus { @@ -37,21 +37,21 @@ pub struct BlockQueueStatus { pub type TreeRoute = ::blockchain::TreeRoute; -pub type BlockNumber = u32; +pub type BlockNumber = u64; pub trait BlockChainClient : Sync { - fn block_header(&self, h: &H256) -> Option; - fn block_body(&self, h: &H256) -> Option; - fn block(&self, h: &H256) -> Option; - fn block_status(&self, h: &H256) -> BlockStatus; + fn block_header(&self, hash: &H256) -> Option; + fn block_body(&self, hash: &H256) -> Option; + fn block(&self, hash: &H256) -> Option; + fn block_status(&self, hash: &H256) -> BlockStatus; fn block_header_at(&self, n: BlockNumber) -> Option; fn block_body_at(&self, n: BlockNumber) -> Option; fn block_at(&self, n: BlockNumber) -> Option; fn block_status_at(&self, n: BlockNumber) -> BlockStatus; fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; - fn state_data(&self, h: &H256) -> Option; - fn block_receipts(&self, h: &H256) -> Option; - fn import_block(&mut self, b: &[u8]) -> ImportResult; + fn state_data(&self, hash: &H256) -> Option; + fn block_receipts(&self, hash: &H256) -> Option; + fn import_block(&mut self, byte: &[u8]) -> ImportResult; fn queue_status(&self) -> BlockQueueStatus; fn clear_queue(&mut self); fn info(&self) -> BlockChainInfo; diff --git a/src/lib.rs b/src/lib.rs index df0b0d8b6..0268b9935 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,6 +104,8 @@ pub mod genesis; pub mod views; pub mod blockchain; pub mod extras; +pub mod client; pub mod eth; pub mod sync; + diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 5d89115e3..4d2a6f990 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -202,7 +202,7 @@ impl ChainSync { self.highest_block = 0; self.have_common_block = false; io.chain().clear_queue(); - self.starting_block = io.chain().info().last_block_number; + self.starting_block = io.chain().info().best_block_number; self.state = SyncState::NotSynced; } @@ -516,7 +516,7 @@ impl ChainSync { if !self.have_common_block { // download backwards until common block is found 1 header at a time let chain_info = io.chain().info(); - start = chain_info.last_block_number as usize; + start = chain_info.best_block_number as usize; if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); } @@ -724,7 +724,7 @@ impl ChainSync { packet.append(&(PROTOCOL_VERSION as u32)); packet.append(&0u32); //TODO: network id packet.append(&chain.total_difficulty); - packet.append(&chain.last_block_hash); + packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); //TODO: handle timeout for status request match io.send(*peer_id, STATUS_PACKET, packet.out()) { @@ -742,7 +742,7 @@ impl ChainSync { let max_headers: usize = r.val_at(1); let skip: usize = r.val_at(2); let reverse: bool = r.val_at(3); - let last = io.chain().info().last_block_number; + let last = io.chain().info().best_block_number; let mut number = if r.at(0).size() == 32 { // id is a hash let hash: H256 = r.val_at(0); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 03a03f49d..a72aa5245 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -63,6 +63,13 @@ pub struct EthSync { pub use self::chain::SyncStatus; impl EthSync { + pub fn new(chain: Arc) -> EthSync { + EthSync { + chain: chain, + sync: ChainSync::new(), + } + } + pub fn is_syncing(&self) -> bool { self.sync.is_syncing() } diff --git a/src/sync/range_collection.rs b/src/sync/range_collection.rs index 5d19d0fd8..5aeadf75f 100644 --- a/src/sync/range_collection.rs +++ b/src/sync/range_collection.rs @@ -180,7 +180,7 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + fn test_range() { use std::cmp::{Ordering}; - let mut ranges: Vec<(u32, Vec)> = Vec::new(); + let mut ranges: Vec<(u64, Vec)> = Vec::new(); assert_eq!(ranges.range_iter().next(), None); assert_eq!(ranges.find_item(&1), None); assert!(!ranges.have_item(&1)); diff --git a/src/sync/tests.rs b/src/sync/tests.rs index e5c8dfe43..7902d2270 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -169,8 +169,8 @@ impl BlockChainClient for TestBlockChainClient { total_difficulty: self.difficulty, pending_total_difficulty: self.difficulty, genesis_hash: self.genesis_hash.clone(), - last_block_hash: self.last_hash.clone(), - last_block_number: self.blocks.len() as BlockNumber - 1, + best_block_hash: self.last_hash.clone(), + best_block_number: self.blocks.len() as BlockNumber - 1, } } } From 24ecd0091a0413d74099fae1c5ee9ac3861c6e1a Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 20:43:37 +0100 Subject: [PATCH 10/27] Client consolidation and docs --- src/eth.rs | 58 ----------------------------------------------- src/lib.rs | 1 - src/sync/chain.rs | 12 +++++----- src/sync/mod.rs | 6 ++--- src/sync/tests.rs | 4 ++-- 5 files changed, 11 insertions(+), 70 deletions(-) delete mode 100644 src/eth.rs diff --git a/src/eth.rs b/src/eth.rs deleted file mode 100644 index 97183d983..000000000 --- a/src/eth.rs +++ /dev/null @@ -1,58 +0,0 @@ -use util::hash::H256; -use util::bytes::Bytes; -use util::uint::U256; - -pub enum QueueStatus { - /// Part of the known chain - Known, - /// Part of the unknown chain - Unknown, -} - -pub enum BlockStatus { - InChain, - Queued(QueueStatus), - Bad, - Unknown, -} - -pub enum ImportResult { - Queued(QueueStatus), - AlreadyInChain, - AlreadyQueued(QueueStatus), - Bad, -} - -pub struct BlockChainInfo { - pub total_difficulty: U256, - pub pending_total_difficulty: U256, - pub genesis_hash: H256, - pub best_block_hash: H256, - pub best_block_number: BlockNumber -} - -pub struct BlockQueueStatus { - pub full: bool, -} - -pub type TreeRoute = ::blockchain::TreeRoute; - -pub type BlockNumber = u64; - -pub trait BlockChainClient : Sync { - fn block_header(&self, hash: &H256) -> Option; - fn block_body(&self, hash: &H256) -> Option; - fn block(&self, hash: &H256) -> Option; - fn block_status(&self, hash: &H256) -> BlockStatus; - fn block_header_at(&self, n: BlockNumber) -> Option; - fn block_body_at(&self, n: BlockNumber) -> Option; - fn block_at(&self, n: BlockNumber) -> Option; - fn block_status_at(&self, n: BlockNumber) -> BlockStatus; - fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; - fn state_data(&self, hash: &H256) -> Option; - fn block_receipts(&self, hash: &H256) -> Option; - fn import_block(&mut self, byte: &[u8]) -> ImportResult; - fn queue_status(&self) -> BlockQueueStatus; - fn clear_queue(&mut self); - fn info(&self) -> BlockChainInfo; -} diff --git a/src/lib.rs b/src/lib.rs index 0268b9935..1d351444c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -106,6 +106,5 @@ pub mod blockchain; pub mod extras; pub mod client; -pub mod eth; pub mod sync; diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 4d2a6f990..420571af8 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -8,7 +8,7 @@ use util::uint::{U256}; use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp use util::rlp::rlptraits::{Stream, View}; use util::sha3::Hashable; -use eth::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; +use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; use views::{HeaderView}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; use sync::{SyncIo}; @@ -202,7 +202,7 @@ impl ChainSync { self.highest_block = 0; self.have_common_block = false; io.chain().clear_queue(); - self.starting_block = io.chain().info().best_block_number; + self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -461,7 +461,7 @@ impl ChainSync { (peer.latest.clone(), peer.difficulty.clone()) }; - let td = io.chain().info().pending_total_difficulty; + let td = io.chain().chain_info().pending_total_difficulty; let syncing_difficulty = max(self.syncing_difficulty, td); if force || peer_difficulty > syncing_difficulty { // start sync @@ -515,7 +515,7 @@ impl ChainSync { let mut start = 0usize; if !self.have_common_block { // download backwards until common block is found 1 header at a time - let chain_info = io.chain().info(); + let chain_info = io.chain().chain_info(); start = chain_info.best_block_number as usize; if !self.headers.is_empty() { start = min(start, self.headers.range_iter().next().unwrap().0 as usize - 1); @@ -720,7 +720,7 @@ impl ChainSync { fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { let mut packet = RlpStream::new_list(5); - let chain = io.chain().info(); + let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); packet.append(&0u32); //TODO: network id packet.append(&chain.total_difficulty); @@ -742,7 +742,7 @@ impl ChainSync { let max_headers: usize = r.val_at(1); let skip: usize = r.val_at(2); let reverse: bool = r.val_at(3); - let last = io.chain().info().best_block_number; + let last = io.chain().chain_info().best_block_number; let mut number = if r.at(0).size() == 32 { // id is a hash let hash: H256 = r.val_at(0); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index a72aa5245..e468b79ee 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,7 +1,7 @@ -use std::sync::{Arc}; -use eth::{BlockChainClient}; +use std::sync::Arc; +use client::BlockChainClient; use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, PacketId, Message, Error as NetworkError}; -use sync::chain::{ChainSync}; +use sync::chain::ChainSync; mod chain; mod range_collection; diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 7902d2270..7651b9298 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -5,7 +5,7 @@ use util::uint::{U256}; use util::sha3::Hashable; use util::rlp::{self, Rlp, RlpStream, View, Stream}; use util::network::{PeerId, PacketId, Error as NetworkError}; -use eth::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; +use client::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; use header::Header as BlockHeader; use sync::{SyncIo}; use sync::chain::{ChainSync}; @@ -164,7 +164,7 @@ impl BlockChainClient for TestBlockChainClient { fn clear_queue(&mut self) { } - fn info(&self) -> BlockChainInfo { + fn chain_info(&self) -> BlockChainInfo { BlockChainInfo { total_difficulty: self.difficulty, pending_total_difficulty: self.difficulty, From edea8d56e3400f6e834efd840126de90df968d0d Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 20:49:36 +0100 Subject: [PATCH 11/27] TODO comment --- src/bin/client.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bin/client.rs b/src/bin/client.rs index e25248673..6809a8229 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -13,6 +13,7 @@ use ethcore::sync::EthSync; fn main() { let mut service = NetworkService::start().unwrap(); + //TODO: replace with proper genesis and chain params. let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); let mut dir = env::temp_dir(); dir.push(H32::random().hex()); From 9b28d4cff6f05b30cea08f3dfc4d9d165fb5f578 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 21:27:41 +0100 Subject: [PATCH 12/27] Removed dup file --- src/sync/range_collection | 136 -------------------------------------- 1 file changed, 136 deletions(-) delete mode 100644 src/sync/range_collection diff --git a/src/sync/range_collection b/src/sync/range_collection deleted file mode 100644 index cc1ccdf01..000000000 --- a/src/sync/range_collection +++ /dev/null @@ -1,136 +0,0 @@ - -pub trait RangeCollection { - fn have_item(&self, key: &K) -> bool; - fn find_item(&self, key: &K) -> Option<&V>; - fn get_tail(&mut self, key: &K) -> Range; - fn remove_head(&mut self, start: &K); - fn remove_tail(&mut self, start: &K); - fn insert_item(&mut self, key: K, value: V); -} - -impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + Add + Sub + Copy + FromUsize + ToUsize { - fn have_item(&self, key: &K) -> bool { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(_) => true, - Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) => k <= key && (*k + FromUsize::from_usize(v.len())) > *key, - _ => false - }, - } - } - - fn find_item(&self, key: &K) -> Option<&V> { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => self.get(index).unwrap().1.get(0), - Err(index) => match self.get(index + 1) { - Some(&(ref k, ref v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => v.get((*key - *k).to_usize()), - _ => None - }, - } - } - - /// Get a range of elements from start till the end of the range - fn get_tail(&mut self, key: &K) -> Range { - let kv = *key; - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => kv..(kv + FromUsize::from_usize(self[index].1.len())), - Err(index) => { - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { - kv..(*k + FromUsize::from_usize(v.len())) - } - _ => kv..kv - } - }, - } - } - /// Remove element key and following elements in the same range - fn remove_tail(&mut self, key: &K) { - match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { - Ok(index) => { self.remove(index); }, - Err(index) =>{ - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref k, ref mut v)) if k <= key && (*k + FromUsize::from_usize(v.len())) > *key => { - v.truncate((*key - *k).to_usize()); - empty = v.is_empty(); - } - _ => {} - } - if empty { - self.remove(index + 1); - } - }, - } - } - - /// Remove range elements up to key - fn remove_head(&mut self, key: &K) { - if *key == FromUsize::from_usize(0) { - return - } - - let prev = *key - FromUsize::from_usize(1); - match self.binary_search_by(|&(k, _)| k.cmp(&prev).reverse()) { - Ok(index) => { self.remove(index); }, - Err(index) => { - let mut empty = false; - match self.get_mut(index + 1) { - Some(&mut (ref mut k, ref mut v)) if *k <= prev && (*k + FromUsize::from_usize(v.len())) > *key => { - let head = v.split_off((*key - *k).to_usize()); - empty = head.is_empty(); - let removed = ::std::mem::replace(v, head); - let new_k = *k - FromUsize::from_usize(removed.len()); - ::std::mem::replace(k, new_k); - } - _ => {} - } - if empty { - self.remove(index + 1); - } - }, - } - } - - fn insert_item(&mut self, key: K, value: V) { - assert!(!self.have_item(&key)); - - let mut lower = match self.binary_search_by(|&(k, _)| k.cmp(&key).reverse()) { - Ok(index) => index, - Err(index) => index, - }; - - lower += 1; - - let mut to_remove: Option = None; - if lower < self.len() && self[lower].0 + FromUsize::from_usize(self[lower].1.len()) == key { - // extend into existing chunk - self[lower].1.push(value); - } - else { - // insert a new chunk - let mut range: Vec = vec![value]; - self.insert(lower, (key, range)); - }; - let next = lower - 1; - if next < self.len() - { - { - let (mut next, mut inserted) = self.split_at_mut(lower); - let mut next = next.last_mut().unwrap(); - let mut inserted = inserted.first_mut().unwrap(); - if next.0 == key + FromUsize::from_usize(1) - { - inserted.1.append(&mut next.1); - to_remove = Some(lower - 1); - } - } - - if let Some(r) = to_remove { - self.remove(r); - } - } - } -} - From 68cd250ec02dfc4fb00849911d8f326b45a0c798 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 21:28:17 +0100 Subject: [PATCH 13/27] Fixed block header seal fields decoding --- src/header.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/header.rs b/src/header.rs index 2a581f6ce..896fe2bd5 100644 --- a/src/header.rs +++ b/src/header.rs @@ -96,7 +96,7 @@ impl Decodable for Header { }; for i in 13..r.item_count() { - blockheader.seal.push(try!(r.val_at(i))) + blockheader.seal.push(try!(r.at(i)).raw().to_vec()) } Ok(blockheader) From 5ef719ae23c15f3bb79f7e64207606507004d6bf Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 7 Jan 2016 21:35:06 +0100 Subject: [PATCH 14/27] Missing client module --- src/client.rs | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 src/client.rs diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 000000000..76c2bac01 --- /dev/null +++ b/src/client.rs @@ -0,0 +1,215 @@ +use std::path::Path; +use util::uint::U256; +use util::hash::*; +use util::sha3::*; +use util::rlp::*; +use util::bytes::Bytes; +use blockchain::BlockChain; +use views::BlockView; + +/// Status for a block in a queue. +pub enum QueueStatus { + /// Part of the known chain. + Known, + /// Part of the unknown chain. + Unknown, +} + +/// General block status +pub enum BlockStatus { + /// Part of the blockchain. + InChain, + /// Queued for import. + Queued(QueueStatus), + /// Known as bad. + Bad, + /// Unknown. + Unknown, +} + +/// Result of import block operation. +pub enum ImportResult { + /// Added to import queue. + Queued(QueueStatus), + /// Already in the chain. + AlreadyInChain, + /// Already queued for import. + AlreadyQueued(QueueStatus), + /// Bad or already known as bad. + Bad, +} + +/// Information about the blockchain gthered together. +pub struct BlockChainInfo { + /// Blockchain difficulty. + pub total_difficulty: U256, + /// Block queue difficulty. + pub pending_total_difficulty: U256, + /// Genesis block hash. + pub genesis_hash: H256, + /// Best blockchain block hash. + pub best_block_hash: H256, + /// Best blockchain block number. + pub best_block_number: BlockNumber +} + +/// Block queue status +pub struct BlockQueueStatus { + pub full: bool, +} + +pub type TreeRoute = ::blockchain::TreeRoute; + +pub type BlockNumber = u64; + +/// Blockchain database client. Owns and manages a blockchain and a block queue. +pub trait BlockChainClient : Sync { + /// Get raw block header data by block header hash. + fn block_header(&self, hash: &H256) -> Option; + + /// Get raw block body data by block header hash. + /// Block body is an RLP list of two items: uncles and transactions. + fn block_body(&self, hash: &H256) -> Option; + + /// Get raw block data by block header hash. + fn block(&self, hash: &H256) -> Option; + + /// Get block status by block header hash. + fn block_status(&self, hash: &H256) -> BlockStatus; + + /// Get raw block header data by block number. + fn block_header_at(&self, n: BlockNumber) -> Option; + + /// Get raw block body data by block number. + /// Block body is an RLP list of two items: uncles and transactions. + fn block_body_at(&self, n: BlockNumber) -> Option; + + /// Get raw block data by block number. + fn block_at(&self, n: BlockNumber) -> Option; + + /// Get block status by block number. + fn block_status_at(&self, n: BlockNumber) -> BlockStatus; + + /// Get a tree route between `from` and `to`. + /// See `BlockChain::tree_route`. + fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; + + /// Get latest state node + fn state_data(&self, hash: &H256) -> Option; + + /// Get raw block receipts data by block header hash. + fn block_receipts(&self, hash: &H256) -> Option; + + /// Import a block into the blockchain. + fn import_block(&mut self, byte: &[u8]) -> ImportResult; + + /// Get block queue information. + fn queue_status(&self) -> BlockQueueStatus; + + /// Clear block qeueu and abort all import activity. + fn clear_queue(&mut self); + + /// Get blockchain information. + fn chain_info(&self) -> BlockChainInfo; +} + +/// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. +pub struct Client { + chain: BlockChain +} + +impl Client { + pub fn new(genesis: &[u8], path: &Path) -> Client { + Client { + chain: BlockChain::new(genesis, path) + } + } +} + +impl BlockChainClient for Client { + fn block_header(&self, hash: &H256) -> Option { + self.chain.block(hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).raw().to_vec()) + } + + fn block_body(&self, hash: &H256) -> Option { + self.chain.block(hash).map(|bytes| { + let rlp = Rlp::new(&bytes); + let mut body = RlpStream::new(); + body.append_raw(rlp.at(1).raw(), 1); + body.append_raw(rlp.at(2).raw(), 1); + body.out() + }) + } + + fn block(&self, hash: &H256) -> Option { + self.chain.block(hash) + } + + fn block_status(&self, hash: &H256) -> BlockStatus { + if self.chain.is_known(&hash) { BlockStatus::InChain } else { BlockStatus::Unknown } + } + + fn block_header_at(&self, n: BlockNumber) -> Option { + self.chain.block_hash(&From::from(n)).and_then(|h| self.block_header(&h)) + } + + fn block_body_at(&self, n: BlockNumber) -> Option { + self.chain.block_hash(&From::from(n)).and_then(|h| self.block_body(&h)) + } + + fn block_at(&self, n: BlockNumber) -> Option { + self.chain.block_hash(&From::from(n)).and_then(|h| self.block(&h)) + } + + fn block_status_at(&self, n: BlockNumber) -> BlockStatus { + match self.chain.block_hash(&From::from(n)) { + Some(h) => self.block_status(&h), + None => BlockStatus::Unknown + } + } + + fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute { + self.chain.tree_route(from.clone(), to.clone()) + } + + fn state_data(&self, _hash: &H256) -> Option { + unimplemented!(); + } + + fn block_receipts(&self, _hash: &H256) -> Option { + unimplemented!(); + } + + fn import_block(&mut self, bytes: &[u8]) -> ImportResult { + //TODO: verify block + { + let block = BlockView::new(bytes); + let header = block.header_view(); + let hash = header.sha3(); + if self.chain.is_known(&hash) { + return ImportResult::Bad; + } + } + self.chain.insert_block(bytes); + ImportResult::Queued(QueueStatus::Known) + } + + fn queue_status(&self) -> BlockQueueStatus { + BlockQueueStatus { + full: false + } + } + + fn clear_queue(&mut self) { + } + + fn chain_info(&self) -> BlockChainInfo { + BlockChainInfo { + total_difficulty: self.chain.best_block_total_difficulty(), + pending_total_difficulty: self.chain.best_block_total_difficulty(), + genesis_hash: self.chain.genesis_hash(), + best_block_hash: self.chain.best_block_hash(), + best_block_number: From::from(self.chain.best_block_number()) + } + } +} From 290d738e3f23da0172fcf5eaf85c1a60d2cdb798 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 16:00:32 +0100 Subject: [PATCH 15/27] Style --- src/client.rs | 6 +++--- src/header.rs | 6 +++--- src/spec.rs | 2 +- src/sync/chain.rs | 16 ++++++++-------- src/sync/tests.rs | 12 ++++++------ src/views.rs | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/client.rs b/src/client.rs index 76c2bac01..f67eb0905 100644 --- a/src/client.rs +++ b/src/client.rs @@ -128,15 +128,15 @@ impl Client { impl BlockChainClient for Client { fn block_header(&self, hash: &H256) -> Option { - self.chain.block(hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).raw().to_vec()) + self.chain.block(hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec()) } fn block_body(&self, hash: &H256) -> Option { self.chain.block(hash).map(|bytes| { let rlp = Rlp::new(&bytes); let mut body = RlpStream::new(); - body.append_raw(rlp.at(1).raw(), 1); - body.append_raw(rlp.at(2).raw(), 1); + body.append_raw(rlp.at(1).as_raw(), 1); + body.append_raw(rlp.at(2).as_raw(), 1); body.out() }) } diff --git a/src/header.rs b/src/header.rs index 896fe2bd5..7f00392ee 100644 --- a/src/header.rs +++ b/src/header.rs @@ -65,7 +65,7 @@ impl Header { hash @ &mut None => { let mut stream = RlpStream::new(); stream.append(self); - let h = stream.raw().sha3(); + let h = stream.as_raw().sha3(); *hash = Some(h.clone()); h.clone() } @@ -92,11 +92,11 @@ impl Decodable for Header { timestamp: try!(r.val_at(11)), extra_data: try!(r.val_at(12)), seal: vec![], - hash: RefCell::new(Some(r.raw().sha3())) + hash: RefCell::new(Some(r.as_raw().sha3())) }; for i in 13..r.item_count() { - blockheader.seal.push(try!(r.at(i)).raw().to_vec()) + blockheader.seal.push(try!(r.at(i)).as_raw().to_vec()) } Ok(blockheader) diff --git a/src/spec.rs b/src/spec.rs index aca29f2e6..371376e16 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -128,7 +128,7 @@ impl Spec { s.out() }; let r = Rlp::new(&seal); - (0..self.seal_fields).map(|i| r.at(i).raw().to_vec()).collect() + (0..self.seal_fields).map(|i| r.at(i).as_raw().to_vec()).collect() }, hash: RefCell::new(None) }; diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 420571af8..1b3ae83fc 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -282,7 +282,7 @@ impl ChainSync { } } let hdr = Header { - data: r.at(i).raw().to_vec(), + data: r.at(i).as_raw().to_vec(), hash: hash.clone(), parent: info.parent_hash(), }; @@ -326,8 +326,8 @@ impl ChainSync { for i in 0..item_count { let body: Rlp = r.at(i); let tx = body.at(0); - let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.raw().to_vec()).collect()); //TODO: get rid of vectors here - let uncles = body.at(1).raw().sha3(); + let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here + let uncles = body.at(1).as_raw().sha3(); let header_id = HeaderId { transactions_root: tx_root, uncles: uncles @@ -335,7 +335,7 @@ impl ChainSync { match self.header_ids.get(&header_id).map(|n| *n) { Some(n) => { self.header_ids.remove(&header_id); - self.bodies.insert_item(n, body.raw().to_vec()); + self.bodies.insert_item(n, body.as_raw().to_vec()); trace!(target: "sync", "Got body {}", n); } None => { @@ -351,10 +351,10 @@ impl ChainSync { fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { let block_rlp = r.at(0); let header_rlp = block_rlp.at(0); - let h = header_rlp.raw().sha3(); + let h = header_rlp.as_raw().sha3(); trace!(target: "sync", "{}-> NewBlock ({})", peer_id, h); - match io.chain().import_block(block_rlp.raw()) { + match io.chain().import_block(block_rlp.as_raw()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "New block already in chain {:?}", h); }, @@ -590,8 +590,8 @@ impl ChainSync { let mut block_rlp = RlpStream::new_list(3); block_rlp.append_raw(&headers.1[i].data, 1); let body = Rlp::new(&bodies.1[i]); - block_rlp.append_raw(body.at(0).raw(), 1); - block_rlp.append_raw(body.at(1).raw(), 1); + block_rlp.append_raw(body.at(0).as_raw(), 1); + block_rlp.append_raw(body.at(1).as_raw(), 1); let h = &headers.1[i].hash; match io.chain().import_block(&block_rlp.out()) { ImportResult::AlreadyInChain => { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 7651b9298..79e1443c9 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -42,28 +42,28 @@ impl TestBlockChainClient { let mut uncles = RlpStream::new_list(if empty {0} else {1}); if !empty { uncles.append(&H256::from(&U256::from(n))); - header.uncles_hash = uncles.raw().sha3(); + header.uncles_hash = uncles.as_raw().sha3(); } let mut rlp = RlpStream::new_list(3); rlp.append(&header); rlp.append_raw(&rlp::NULL_RLP, 1); - rlp.append_raw(uncles.raw(), 1); - self.import_block(rlp.raw()); + rlp.append_raw(uncles.as_raw(), 1); + self.import_block(rlp.as_raw()); } } } impl BlockChainClient for TestBlockChainClient { fn block_header(&self, h: &H256) -> Option { - self.blocks.get(h).map(|r| Rlp::new(r).at(0).raw().to_vec()) + self.blocks.get(h).map(|r| Rlp::new(r).at(0).as_raw().to_vec()) } fn block_body(&self, h: &H256) -> Option { self.blocks.get(h).map(|r| { let mut stream = RlpStream::new_list(2); - stream.append_raw(Rlp::new(&r).at(1).raw(), 1); - stream.append_raw(Rlp::new(&r).at(2).raw(), 1); + stream.append_raw(Rlp::new(&r).at(1).as_raw(), 1); + stream.append_raw(Rlp::new(&r).at(2).as_raw(), 1); stream.out() }) } diff --git a/src/views.rs b/src/views.rs index 8f04d5e82..9d66bb164 100644 --- a/src/views.rs +++ b/src/views.rs @@ -49,7 +49,7 @@ impl<'a> BlockView<'a> { /// Return transaction hashes. pub fn transaction_hashes(&self) -> Vec { - self.rlp.at(1).iter().map(|rlp| rlp.raw().sha3()).collect() + self.rlp.at(1).iter().map(|rlp| rlp.as_raw().sha3()).collect() } /// Return list of uncles of given block. @@ -59,7 +59,7 @@ impl<'a> BlockView<'a> { /// Return list of uncle hashes of given block. pub fn uncle_hashes(&self) -> Vec { - self.rlp.at(2).iter().map(|rlp| rlp.raw().sha3()).collect() + self.rlp.at(2).iter().map(|rlp| rlp.as_raw().sha3()).collect() } } @@ -143,6 +143,6 @@ impl<'a> HeaderView<'a> { impl<'a> Hashable for HeaderView<'a> { fn sha3(&self) -> H256 { - self.rlp.raw().sha3() + self.rlp.as_raw().sha3() } } From 9ea7f145429ad86d4bdd5eb6043a53124cf99572 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 16:26:00 +0100 Subject: [PATCH 16/27] Style --- src/sync/chain.rs | 43 +++++++++++++++++++----------------- src/sync/mod.rs | 6 ++--- src/sync/range_collection.rs | 1 + 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 1b3ae83fc..0d45cda5e 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -81,15 +81,14 @@ pub struct SyncStatus { pub state: SyncState, pub protocol_version: u8, pub start_block_number: BlockNumber, - pub current_block_number: BlockNumber, + pub last_imported_block_number: BlockNumber, pub highest_block_number: BlockNumber, pub blocks_total: usize, - pub blocks_received: usize + pub blocks_received: usize, } #[derive(PartialEq, Eq, Debug)] -enum PeerAsking -{ +enum PeerAsking { Nothing, BlockHeaders, BlockBodies, @@ -102,7 +101,7 @@ struct PeerInfo { latest: H256, difficulty: U256, asking: PeerAsking, - asking_blocks: Vec + asking_blocks: Vec, } type Body = Bytes; @@ -119,9 +118,9 @@ pub struct ChainSync { /// Set of block body numbers being downloaded downloading_bodies: HashSet, /// Downloaded headers. - headers: Vec<(BlockNumber, Vec
)>, //TODO: use BTreeMap once range API is sable. For now it a vector sorted in descending order + headers: Vec<(BlockNumber, Vec
)>, //TODO: use BTreeMap once range API is sable. For now it is a vector sorted in descending order /// Downloaded bodies - bodies: Vec<(BlockNumber, Vec)>, //TODO: use BTreeMap once range API is sable. For now it a vector sorted in descending order + bodies: Vec<(BlockNumber, Vec)>, //TODO: use BTreeMap once range API is sable. For now it is a vector sorted in descending order /// Peer info peers: HashMap, /// Used to map body to header @@ -152,7 +151,7 @@ impl ChainSync { last_imported_block: 0, last_imported_hash: H256::new(), syncing_difficulty: U256::from(0u64), - have_common_block: false + have_common_block: false, } } @@ -162,10 +161,10 @@ impl ChainSync { state: self.state.clone(), protocol_version: 63, start_block_number: self.starting_block, - current_block_number: 0, //TODO: + last_imported_block_number: self.last_imported_block, highest_block_number: self.highest_block, blocks_total: (self.last_imported_block - self.starting_block) as usize, - blocks_received: (self.highest_block - self.starting_block) as usize + blocks_received: (self.highest_block - self.starting_block) as usize, } } @@ -228,16 +227,16 @@ impl ChainSync { } /// Called by peer once it has new block headers during sync - fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { self.reset_peer_asking(peer_id, PeerAsking::BlockHeaders); let item_count = r.item_count(); - trace!(target: "sync", "{}-> BlockHeaders ({} entries)", peer_id, item_count); + trace!(target: "sync", "{} -> BlockHeaders ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block headers"); return; } - if self.state == SyncState::Waiting { + if self.state == SyncState::Waiting { trace!(target: "sync", "Ignored block headers while waiting"); return; } @@ -313,7 +312,7 @@ impl ChainSync { fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { self.reset_peer_asking(peer_id, PeerAsking::BlockBodies); let item_count = r.item_count(); - trace!(target: "sync", "{}-> BlockBodies ({} entries)", peer_id, item_count); + trace!(target: "sync", "{} -> BlockBodies ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block bodies"); @@ -353,7 +352,7 @@ impl ChainSync { let header_rlp = block_rlp.at(0); let h = header_rlp.as_raw().sha3(); - trace!(target: "sync", "{}-> NewBlock ({})", peer_id, h); + trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h); match io.chain().import_block(block_rlp.as_raw()) { ImportResult::AlreadyInChain => { trace!(target: "sync", "New block already in chain {:?}", h); @@ -386,7 +385,7 @@ impl ChainSync { trace!(target: "sync", "Ignoring new hashes since we're already downloading."); return; } - trace!(target: "sync", "{}-> NewHashes ({} entries)", peer_id, r.item_count()); + trace!(target: "sync", "{} -> NewHashes ({} entries)", peer_id, r.item_count()); let hashes = r.iter().map(|item| (item.val_at::(0), item.val_at::(1))); let mut max_height: U256 = From::from(0); for (h, d) in hashes { @@ -491,7 +490,7 @@ impl ChainSync { if self.have_common_block && !self.headers.is_empty() && self.headers.range_iter().next().unwrap().0 == self.last_imported_block + 1 { for (start, ref items) in self.headers.range_iter() { - if needed_bodies.len() > MAX_BODIES_TO_REQUEST { + if needed_bodies.len() > MAX_BODIES_TO_REQUEST { break; } let mut index: BlockNumber = 0; @@ -732,7 +731,7 @@ impl ChainSync { warn!(target:"sync", "Error sending status request: {:?}", e); io.disable_peer(peer_id); } - Ok(_) => { } + Ok(_) => () } } @@ -765,6 +764,7 @@ impl ChainSync { let max_count = min(MAX_HEADERS_TO_SEND, max_headers); let mut count = 0; let mut data = Bytes::new(); + let inc = (skip + 1) as BlockNumber; while number <= last && number > 0 && count < max_count { match io.chain().block_header_at(number) { Some(mut hdr) => { @@ -774,10 +774,13 @@ impl ChainSync { None => {} } if reverse { - number -= (skip + 1) as BlockNumber; + if number <= inc { + break; + } + number -= inc; } else { - number += (skip + 1) as BlockNumber; + number += inc; } } let mut rlp = RlpStream::new_list(count as usize); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index e468b79ee..fc3553180 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -9,7 +9,7 @@ mod range_collection; #[cfg(test)] mod tests; -pub fn new(_service: &mut NetworkService, eth_client: Arc) -> EthSync { +pub fn new(_service: &mut NetworkService, eth_client: Arc) -> EthSync { EthSync { chain: eth_client, sync: ChainSync::new(), @@ -56,14 +56,14 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { } pub struct EthSync { - chain: Arc, + chain: Arc, sync: ChainSync } pub use self::chain::SyncStatus; impl EthSync { - pub fn new(chain: Arc) -> EthSync { + pub fn new(chain: Arc) -> EthSync { EthSync { chain: chain, sync: ChainSync::new(), diff --git a/src/sync/range_collection.rs b/src/sync/range_collection.rs index 5aeadf75f..822e88d75 100644 --- a/src/sync/range_collection.rs +++ b/src/sync/range_collection.rs @@ -205,6 +205,7 @@ fn test_range() { assert_eq!(ranges.find_item(&18), Some(&'r')); assert!(ranges.have_item(&16)); assert_eq!(ranges.get_tail(&17), 17..19); + assert_eq!(ranges.get_tail(&16), 16..19); ranges.insert_item(2, 'b'); assert_eq!(ranges.range_iter().cmp(vec![(2, &['b'][..]), (16, &['p', 'q', 'r'][..])]), Ordering::Equal); From b27259518389697128e7219224c4d3f0ab005a7f Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 17:52:25 +0100 Subject: [PATCH 17/27] Rlp error handling --- src/sync/chain.rs | 143 +++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 60 deletions(-) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 0d45cda5e..5d0798c80 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -5,11 +5,13 @@ use util::network::{PeerId, PacketId}; use util::hash::{H256, FixedHash}; use util::bytes::{Bytes}; use util::uint::{U256}; -use util::rlp::{Rlp, RlpStream, self}; //TODO: use UntrustedRlp +use util::rlp::{Rlp, UntrustedRlp, RlpStream, self}; use util::rlp::rlptraits::{Stream, View}; +use util::rlp::rlperrors::DecoderError; use util::sha3::Hashable; use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; use views::{HeaderView}; +use header::{Header as BlockHeader}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; use sync::{SyncIo}; @@ -25,6 +27,8 @@ impl FromUsize for BlockNumber { } } +type PacketDecodeError = DecoderError; + const PROTOCOL_VERSION: u8 = 63u8; const MAX_BODIES_TO_SEND: usize = 256; const MAX_HEADERS_TO_SEND: usize = 512; @@ -206,13 +210,13 @@ impl ChainSync { } /// Called by peer to report status - fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let peer = PeerInfo { - protocol_version: r.val_at(0), - network_id: r.val_at(1), - difficulty: r.val_at(2), - latest: r.val_at(3), - genesis: r.val_at(4), + protocol_version: try!(r.val_at(0)), + network_id: try!(r.val_at(1)), + difficulty: try!(r.val_at(2)), + latest: try!(r.val_at(3)), + genesis: try!(r.val_at(4)), asking: PeerAsking::Nothing, asking_blocks: Vec::new(), }; @@ -224,26 +228,27 @@ impl ChainSync { panic!("ChainSync: new peer already exists"); } self.sync_peer(io, peer_id, false); + Ok(()) } /// Called by peer once it has new block headers during sync - fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { self.reset_peer_asking(peer_id, PeerAsking::BlockHeaders); let item_count = r.item_count(); trace!(target: "sync", "{} -> BlockHeaders ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block headers"); - return; + return Ok(()); } if self.state == SyncState::Waiting { trace!(target: "sync", "Ignored block headers while waiting"); - return; + return Ok(()); } for i in 0..item_count { - let info = HeaderView::new_from_rlp(r.at(i)); - let number = BlockNumber::from(info.number()); + let info: BlockHeader = try!(r.val_at(i)); + let number = BlockNumber::from(info.number); if number <= self.last_imported_block || self.headers.have_item(&number) { trace!(target: "sync", "Skipping existing block header"); continue; @@ -251,7 +256,7 @@ impl ChainSync { if number > self.highest_block { self.highest_block = number; } - let hash = info.sha3(); + let hash = info.hash(); match io.chain().block_status(&hash) { BlockStatus::InChain => { self.have_common_block = true; @@ -262,12 +267,12 @@ impl ChainSync { _ => { if self.have_common_block { //validate chain - if self.have_common_block && number == self.last_imported_block + 1 && info.parent_hash() != self.last_imported_hash { + if self.have_common_block && number == self.last_imported_block + 1 && info.parent_hash != self.last_imported_hash { // TODO: lower peer rating debug!(target: "sync", "Mismatched block header {} {}", number, hash); continue; } - if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash()) { + if self.headers.find_item(&(number - 1)).map_or(false, |p| p.hash != info.parent_hash) { // mismatching parent id, delete the previous block and don't add this one // TODO: lower peer rating debug!(target: "sync", "Mismatched block header {} {}", number, hash); @@ -281,14 +286,14 @@ impl ChainSync { } } let hdr = Header { - data: r.at(i).as_raw().to_vec(), + data: try!(r.at(i)).as_raw().to_vec(), hash: hash.clone(), - parent: info.parent_hash(), + parent: info.parent_hash, }; self.headers.insert_item(number, hdr); let header_id = HeaderId { - transactions_root: info.transactions_root(), - uncles: info.uncles_hash() + transactions_root: info.transactions_root, + uncles: info.uncles_hash }; trace!(target: "sync", "Got header {} ({})", number, hash); if header_id.transactions_root == rlp::SHA3_NULL_RLP && header_id.uncles == rlp::SHA3_EMPTY_LIST_RLP { @@ -306,27 +311,29 @@ impl ChainSync { } self.collect_blocks(io); self.continue_sync(io); + Ok(()) } /// Called by peer once it has new block bodies - fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + use util::triehash::ordered_trie_root; self.reset_peer_asking(peer_id, PeerAsking::BlockBodies); let item_count = r.item_count(); trace!(target: "sync", "{} -> BlockBodies ({} entries)", peer_id, item_count); self.clear_peer_download(peer_id); if self.state != SyncState::Blocks && self.state != SyncState::NewBlocks && self.state != SyncState::Waiting { trace!(target: "sync", "Ignored unexpected block bodies"); - return; + return Ok(()); } if self.state == SyncState::Waiting { trace!(target: "sync", "Ignored block bodies while waiting"); - return; + return Ok(()); } for i in 0..item_count { - let body: Rlp = r.at(i); - let tx = body.at(0); - let tx_root = ::util::triehash::ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here - let uncles = body.at(1).as_raw().sha3(); + let body = try!(r.at(i)); + let tx = try!(body.at(0)); + let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here + let uncles = try!(body.at(1)).as_raw().sha3(); let header_id = HeaderId { transactions_root: tx_root, uncles: uncles @@ -344,20 +351,21 @@ impl ChainSync { } self.collect_blocks(io); self.continue_sync(io); + Ok(()) } /// Called by peer once it has new block bodies - fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { - let block_rlp = r.at(0); - let header_rlp = block_rlp.at(0); + fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + let block_rlp = try!(r.at(0)); + let header_rlp = try!(block_rlp.at(0)); let h = header_rlp.as_raw().sha3(); trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h); match io.chain().import_block(block_rlp.as_raw()) { - ImportResult::AlreadyInChain => { + ImportResult::AlreadyInChain => { trace!(target: "sync", "New block already in chain {:?}", h); }, - ImportResult::AlreadyQueued(_) => { + ImportResult::AlreadyQueued(_) => { trace!(target: "sync", "New block already queued {:?}", h); }, ImportResult::Queued(QueueStatus::Known) => { @@ -366,7 +374,7 @@ impl ChainSync { ImportResult::Queued(QueueStatus::Unknown) => { trace!(target: "sync", "New block unknown {:?}", h); //TODO: handle too many unknown blocks - let difficulty: U256 = r.val_at(1); + let difficulty: U256 = try!(r.val_at(1)); let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; if difficulty > peer_difficulty { trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); @@ -377,18 +385,21 @@ impl ChainSync { debug!(target: "sync", "Bad new block {:?}", h); io.disable_peer(peer_id); } - } + }; + Ok(()) } - fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &Rlp) { + fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { if self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").asking != PeerAsking::Nothing { trace!(target: "sync", "Ignoring new hashes since we're already downloading."); - return; + return Ok(()); } trace!(target: "sync", "{} -> NewHashes ({} entries)", peer_id, r.item_count()); let hashes = r.iter().map(|item| (item.val_at::(0), item.val_at::(1))); let mut max_height: U256 = From::from(0); - for (h, d) in hashes { + for (rh, rd) in hashes { + let h = try!(rh); + let d = try!(rd); match io.chain().block_status(&h) { BlockStatus::InChain => { trace!(target: "sync", "New block hash already in chain {:?}", h); @@ -407,10 +418,11 @@ impl ChainSync { BlockStatus::Bad =>{ debug!(target: "sync", "Bad new block hash {:?}", h); io.disable_peer(peer_id); - return; + return Ok(()); } } - } + }; + Ok(()) } /// Called by peer when it is disconnecting @@ -714,7 +726,8 @@ impl ChainSync { } } - fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: &PeerId, _r: &Rlp) { + fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: &PeerId, _r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + Ok(()) } fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { @@ -735,16 +748,16 @@ impl ChainSync { } } - fn return_block_headers(&self, io: &mut SyncIo, r: &Rlp) { + fn return_block_headers(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { // Packet layout: // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] - let max_headers: usize = r.val_at(1); - let skip: usize = r.val_at(2); - let reverse: bool = r.val_at(3); + let max_headers: usize = try!(r.val_at(1)); + let skip: usize = try!(r.val_at(2)); + let reverse: bool = try!(r.val_at(3)); let last = io.chain().chain_info().best_block_number; - let mut number = if r.at(0).size() == 32 { + let mut number = if try!(r.at(0)).size() == 32 { // id is a hash - let hash: H256 = r.val_at(0); + let hash: H256 = try!(r.val_at(0)); trace!(target: "sync", "-> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", hash, max_headers, skip, reverse); match io.chain().block_header(&hash) { Some(hdr) => From::from(HeaderView::new(&hdr).number()), @@ -752,8 +765,8 @@ impl ChainSync { } } else { - trace!(target: "sync", "-> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", r.val_at::(0), max_headers, skip, reverse); - r.val_at(0) + trace!(target: "sync", "-> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", try!(r.val_at::(0)), max_headers, skip, reverse); + try!(r.val_at(0)) }; if reverse { @@ -788,20 +801,21 @@ impl ChainSync { io.respond(BLOCK_HEADERS_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); trace!(target: "sync", "-> GetBlockHeaders: returned {} entries", count); + Ok(()) } - fn return_block_bodies(&self, io: &mut SyncIo, r: &Rlp) { + fn return_block_bodies(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); - return; + return Ok(()); } trace!(target: "sync", "-> GetBlockBodies: {} entries", count); count = min(count, MAX_BODIES_TO_SEND); let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain().block_body(&r.val_at::(i)) { + match io.chain().block_body(&try!(r.val_at::(i))) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -814,19 +828,20 @@ impl ChainSync { io.respond(BLOCK_BODIES_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); trace!(target: "sync", "-> GetBlockBodies: returned {} entries", added); + Ok(()) } - fn return_node_data(&self, io: &mut SyncIo, r: &Rlp) { + fn return_node_data(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetNodeData request, ignoring."); - return; + return Ok(()); } count = min(count, MAX_NODE_DATA_TO_SEND); let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain().state_data(&r.val_at::(i)) { + match io.chain().state_data(&try!(r.val_at::(i))) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -838,19 +853,20 @@ impl ChainSync { rlp.append_raw(&data, added); io.respond(NODE_DATA_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); + Ok(()) } - fn return_receipts(&self, io: &mut SyncIo, r: &Rlp) { + fn return_receipts(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { debug!(target: "sync", "Empty GetReceipts request, ignoring."); - return; + return Ok(()); } count = min(count, MAX_RECEIPTS_TO_SEND); let mut added = 0usize; let mut data = Bytes::new(); for i in 0..count { - match io.chain().block_receipts(&r.val_at::(i)) { + match io.chain().block_receipts(&try!(r.val_at::(i))) { Some(mut hdr) => { data.append(&mut hdr); added += 1; @@ -862,11 +878,12 @@ impl ChainSync { rlp.append_raw(&data, added); io.respond(RECEIPTS_PACKET, rlp.out()).unwrap_or_else(|e| debug!(target: "sync", "Error sending headers: {:?}", e)); + Ok(()) } pub fn on_packet(&mut self, io: &mut SyncIo, peer: &PeerId, packet_id: u8, data: &[u8]) { - let rlp = Rlp::new(data); - match packet_id { + let rlp = UntrustedRlp::new(data); + let result = match packet_id { STATUS_PACKET => self.on_peer_status(io, peer, &rlp), TRANSACTIONS_PACKET => self.on_peer_transactions(io, peer, &rlp), GET_BLOCK_HEADERS_PACKET => self.return_block_headers(io, &rlp), @@ -877,8 +894,14 @@ impl ChainSync { NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), GET_NODE_DATA_PACKET => self.return_node_data(io, &rlp), GET_RECEIPTS_PACKET => self.return_receipts(io, &rlp), - _ => debug!(target: "sync", "Unknown packet {}", packet_id) - } + _ => { + debug!(target: "sync", "Unknown packet {}", packet_id); + Ok(()) + } + }; + result.unwrap_or_else(|e| { + debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e); + }) } pub fn maintain_sync(&mut self, _io: &mut SyncIo) { From f1cdc0a17de352f4385f01e1081a79f426b2d0b1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 10:16:35 +0100 Subject: [PATCH 18/27] Started block verification --- src/client.rs | 25 ++++------ src/lib.rs | 3 +- src/queue.rs | 37 +++++++++++++++ src/verification.rs | 109 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 src/queue.rs create mode 100644 src/verification.rs diff --git a/src/client.rs b/src/client.rs index f67eb0905..9b5d2f26e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,10 +1,11 @@ +use std::sync::Arc; use std::path::Path; use util::uint::U256; use util::hash::*; -use util::sha3::*; use util::rlp::*; use util::bytes::Bytes; use blockchain::BlockChain; +use queue::BlockQueue; use views::BlockView; /// Status for a block in a queue. @@ -106,7 +107,7 @@ pub trait BlockChainClient : Sync { /// Get block queue information. fn queue_status(&self) -> BlockQueueStatus; - /// Clear block qeueu and abort all import activity. + /// Clear block queue and abort all import activity. fn clear_queue(&mut self); /// Get blockchain information. @@ -115,13 +116,16 @@ pub trait BlockChainClient : Sync { /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. pub struct Client { - chain: BlockChain + chain: Arc, + queue: BlockQueue, } impl Client { pub fn new(genesis: &[u8], path: &Path) -> Client { + let chain = Arc::new(BlockChain::new(genesis, path)); Client { - chain: BlockChain::new(genesis, path) + chain: chain.clone(), + queue: BlockQueue::new(chain) } } } @@ -181,17 +185,7 @@ impl BlockChainClient for Client { } fn import_block(&mut self, bytes: &[u8]) -> ImportResult { - //TODO: verify block - { - let block = BlockView::new(bytes); - let header = block.header_view(); - let hash = header.sha3(); - if self.chain.is_known(&hash) { - return ImportResult::Bad; - } - } - self.chain.insert_block(bytes); - ImportResult::Queued(QueueStatus::Known) + self.queue.import_block(bytes) } fn queue_status(&self) -> BlockQueueStatus { @@ -201,6 +195,7 @@ impl BlockChainClient for Client { } fn clear_queue(&mut self) { + self.queue.clear(); } fn chain_info(&self) -> BlockChainInfo { diff --git a/src/lib.rs b/src/lib.rs index 1d351444c..80f9b732e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,6 +105,7 @@ pub mod views; pub mod blockchain; pub mod extras; pub mod client; - pub mod sync; +pub mod verification; +pub mod queue; diff --git a/src/queue.rs b/src/queue.rs new file mode 100644 index 000000000..8d427e8a1 --- /dev/null +++ b/src/queue.rs @@ -0,0 +1,37 @@ +use std::sync::Arc; +//use util::bytes::*; +use util::sha3::*; +use blockchain::BlockChain; +use client::{QueueStatus, ImportResult}; +use views::{BlockView}; + + +pub struct BlockQueue { + chain: Arc +} + +impl BlockQueue { + pub fn new(chain: Arc) -> BlockQueue { + BlockQueue { + chain: chain + } + } + + pub fn clear(&mut self) { + } + + pub fn import_block(&mut self, bytes: &[u8]) -> ImportResult { + //TODO: verify block + { + let block = BlockView::new(bytes); + let header = block.header_view(); + let hash = header.sha3(); + if self.chain.is_known(&hash) { + return ImportResult::Bad; + } + } + self.chain.insert_block(bytes); + ImportResult::Queued(QueueStatus::Known) + } +} + diff --git a/src/verification.rs b/src/verification.rs new file mode 100644 index 000000000..95951da53 --- /dev/null +++ b/src/verification.rs @@ -0,0 +1,109 @@ +use util::uint::*; +use util::hash::*; +use util::rlp::*; +use util::sha3::Hashable; +use util::triehash::ordered_trie_root; +use header::Header; +use client::BlockNumber; + +#[derive(Debug)] +pub enum TransactionVerificationError { + OutOfGasBase, + OutOfGasIntrinsic, + NotEnoughCash, + GasPriceTooLow, + BlockGasLimitReached, + FeeTooSmall, + TooMuchGasUsed { + used: U256, + limit: U256 + }, + ExtraDataTooBig, + InvalidSignature, + InvalidTransactionFormat, +} + +#[derive(Debug)] +pub enum BlockVerificationError { + TooMuchGasUsed { + used: U256, + limit: U256, + }, + InvalidBlockFormat, + InvalidUnclesHash { + expected: H256, + got: H256, + }, + TooManyUncles, + UncleTooOld, + UncleIsBrother, + UncleInChain, + UncleParentNotInChain, + InvalidStateRoot, + InvalidGasUsed, + InvalidTransactionsRoot { + expected: H256, + got: H256, + }, + InvalidDifficulty, + InvalidGasLimit, + InvalidReceiptsStateRoot, + InvalidTimestamp, + InvalidLogBloom, + InvalidNonce, + InvalidBlockHeaderItemCount, + InvalidBlockNonce, + InvalidParentHash, + InvalidUncleParentHash, + InvalidNumber, + BlockNotFound, + UnknownParent, +} + + +pub fn verify_header(header: &Header) -> Result<(), BlockVerificationError> { + if header.number > From::from(BlockNumber::max_value()) { + return Err(BlockVerificationError::InvalidNumber) + } + if header.gas_used > header.gas_limit { + return Err(BlockVerificationError::TooMuchGasUsed { + used: header.gas_used, + limit: header.gas_limit, + }); + } + Ok(()) +} + +pub fn verify_parent(header: &Header, parent: &Header) -> Result<(), BlockVerificationError> { + if !header.parent_hash.is_zero() && parent.hash() != header.parent_hash { + return Err(BlockVerificationError::InvalidParentHash) + } + if header.timestamp <= parent.timestamp { + return Err(BlockVerificationError::InvalidTimestamp) + } + if header.number <= parent.number { + return Err(BlockVerificationError::InvalidNumber) + } + Ok(()) +} + +pub fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), BlockVerificationError> { + let block = Rlp::new(block); + let tx = block.at(1); + let expected_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here + if &expected_root != transactions_root { + return Err(BlockVerificationError::InvalidTransactionsRoot { + expected: expected_root.clone(), + got: transactions_root.clone(), + }); + } + let expected_uncles = block.at(2).as_raw().sha3(); + if &expected_uncles != uncles_hash { + return Err(BlockVerificationError::InvalidUnclesHash { + expected: expected_uncles.clone(), + got: uncles_hash.clone(), + }); + } + Ok(()) +} + From 10fe6b937f798eaf53e0e6172155787e2fd45ceb Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 18:40:13 +0100 Subject: [PATCH 19/27] Docs --- src/bin/client.rs | 9 ++-- src/sync/chain.rs | 60 ++++++++++++++++++---- src/sync/mod.rs | 97 +++++++++++++++--------------------- src/sync/range_collection.rs | 16 +++++- src/sync/tests.rs | 4 +- 5 files changed, 112 insertions(+), 74 deletions(-) diff --git a/src/bin/client.rs b/src/bin/client.rs index 6809a8229..97e587989 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -5,21 +5,20 @@ extern crate rustc_serialize; use std::io::*; use std::env; use std::sync::Arc; -use rustc_serialize::hex::FromHex; use util::hash::*; use util::network::{NetworkService}; use ethcore::client::Client; use ethcore::sync::EthSync; +use ethcore::spec::Spec; fn main() { let mut service = NetworkService::start().unwrap(); //TODO: replace with proper genesis and chain params. - let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); + let frontier = Spec::new_frontier(); let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - let client = Arc::new(Client::new(&genesis, &dir)); - let sync = Box::new(EthSync::new(client)); - service.register_protocol(sync, "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); + let client = Arc::new(Client::new(&frontier.genesis_block(), &dir)); + EthSync::register(&mut service, client); loop { let mut cmd = String::new(); stdin().read_line(&mut cmd).unwrap(); diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 5d0798c80..b7d550c9b 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -1,3 +1,18 @@ +/// +/// BlockChain synchronization strategy. +/// Syncs to peers and keeps up to date. +/// This implementation uses ethereum protocol v63 +/// +/// Syncing strategy. +/// +/// 1. A peer arrives with a total difficulty better than ours +/// 2. Find a common best block between our an peer chain. +/// Start with out best block and request headers from peer backwards until a common block is found +/// 3. Download headers and block bodies from peers in parallel. +/// As soon as a set of the blocks is fully downloaded at the head of the queue it is fed to the blockchain +/// 4. Maintain sync by handling NewBlocks/NewHashes messages +/// + use std::collections::{HashSet, HashMap}; use std::cmp::{min, max}; use std::mem::{replace}; @@ -13,7 +28,7 @@ use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResu use views::{HeaderView}; use header::{Header as BlockHeader}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; -use sync::{SyncIo}; +use sync::io::SyncIo; impl ToUsize for BlockNumber { fn to_usize(&self) -> usize { @@ -52,7 +67,7 @@ const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; struct Header { - ///Header data + /// Header data data: Bytes, /// Block hash hash: H256, @@ -81,13 +96,21 @@ pub enum SyncState { NewBlocks, } +/// Syncing status and statistics pub struct SyncStatus { + /// State pub state: SyncState, + /// Syncing protocol version. That's the maximum protocol version we connect to. pub protocol_version: u8, + /// BlockChain height for the moment the sync started. pub start_block_number: BlockNumber, + /// Last fully downloaded and imported block number. pub last_imported_block_number: BlockNumber, + /// Highest block number in the download queue. pub highest_block_number: BlockNumber, + /// Total number of blocks for the sync process. pub blocks_total: usize, + /// Number of blocks downloaded so far. pub blocks_received: usize, } @@ -178,11 +201,7 @@ impl ChainSync { self.peers.clear(); } - /// @returns true is Sync is in progress - pub fn is_syncing(&self) -> bool { - self.state != SyncState::Idle - } - + /// Rest sync. Clear all downloaded data but keep the queue fn reset(&mut self) { self.downloading_headers.clear(); self.downloading_bodies.clear(); @@ -389,6 +408,7 @@ impl ChainSync { Ok(()) } + /// Handles NewHashes packet. Initiates headers download for any unknown hashes. fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer_id: &PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { if self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").asking != PeerAsking::Nothing { trace!(target: "sync", "Ignoring new hashes since we're already downloading."); @@ -432,6 +452,7 @@ impl ChainSync { self.continue_sync(io); } + /// Called when a new peer is connected pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: &PeerId) { trace!(target: "sync", "== Connected {}", peer); self.send_status(io, peer); @@ -459,6 +480,7 @@ impl ChainSync { self.state = SyncState::Waiting; } + /// Find something to do for a peer. Called for a new peer or when a peer is done with it's task. fn sync_peer(&mut self, io: &mut SyncIo, peer_id: &PeerId, force: bool) { let (peer_latest, peer_difficulty) = { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); @@ -488,6 +510,7 @@ impl ChainSync { } } + /// Find some headers or blocks to download for a peer. fn request_blocks(&mut self, io: &mut SyncIo, peer_id: &PeerId) { self.clear_peer_download(peer_id); @@ -572,6 +595,7 @@ impl ChainSync { } } + /// Clear all blocks/headers marked as being downloaded by a peer. fn clear_peer_download(&mut self, peer_id: &PeerId) { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); for b in &peer.asking_blocks { @@ -581,6 +605,7 @@ impl ChainSync { peer.asking_blocks.clear(); } + /// Checks if there are blocks fully downloaded that can be imported into the blockchain and does the import. fn collect_blocks(&mut self, io: &mut SyncIo) { if !self.have_common_block || self.headers.is_empty() || self.bodies.is_empty() { return; @@ -647,6 +672,8 @@ impl ChainSync { } } + /// Remove downloaded bocks/headers starting from specified number. + /// Used to recover from an error and re-download parts of the chain detected as bad. fn remove_downloaded_blocks(&mut self, start: BlockNumber) { for n in self.headers.get_tail(&start) { match self.headers.find_item(&n) { @@ -667,8 +694,9 @@ impl ChainSync { self.bodies.remove_tail(&start); } + /// Request headers from a peer by block hash fn request_headers_by_hash(&mut self, sync: &mut SyncIo, peer_id: &PeerId, h: &H256, count: usize, skip: usize, reverse: bool) { - trace!(target: "sync", "{}<- GetBlockHeaders: {} entries starting from {}", peer_id, count, h); + trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}", peer_id, count, h); let mut rlp = RlpStream::new_list(4); rlp.append(h); rlp.append(&count); @@ -677,9 +705,10 @@ impl ChainSync { self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); } + /// Request headers from a peer by block number fn request_headers_by_number(&mut self, sync: &mut SyncIo, peer_id: &PeerId, n: BlockNumber, count: usize, skip: usize, reverse: bool) { let mut rlp = RlpStream::new_list(4); - trace!(target: "sync", "{}<- GetBlockHeaders: {} entries starting from {}", peer_id, count, n); + trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}", peer_id, count, n); rlp.append(&n); rlp.append(&count); rlp.append(&skip); @@ -687,15 +716,17 @@ impl ChainSync { self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); } + /// Request block bodies from a peer fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: &PeerId, hashes: Vec) { let mut rlp = RlpStream::new_list(hashes.len()); - trace!(target: "sync", "{}<- GetBlockBodies: {} entries", peer_id, hashes.len()); + trace!(target: "sync", "{} <- GetBlockBodies: {} entries", peer_id, hashes.len()); for h in hashes { rlp.append(&h); } self.send_request(sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); } + /// Reset peer status after request is complete. fn reset_peer_asking(&mut self, peer_id: &PeerId, asking: PeerAsking) { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); if peer.asking != asking { @@ -706,6 +737,7 @@ impl ChainSync { } } + /// Generic request sender fn send_request(&mut self, sync: &mut SyncIo, peer_id: &PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { { let peer = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer"); @@ -726,10 +758,12 @@ impl ChainSync { } } + /// Called when peer sends us new transactions fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: &PeerId, _r: &UntrustedRlp) -> Result<(), PacketDecodeError> { Ok(()) } + /// Send Status message fn send_status(&mut self, io: &mut SyncIo, peer_id: &PeerId) { let mut packet = RlpStream::new_list(5); let chain = io.chain().chain_info(); @@ -748,6 +782,7 @@ impl ChainSync { } } + /// Respond to GetBlockHeaders request fn return_block_headers(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { // Packet layout: // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] @@ -804,6 +839,7 @@ impl ChainSync { Ok(()) } + /// Respond to GetBlockBodies request fn return_block_bodies(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { @@ -831,6 +867,7 @@ impl ChainSync { Ok(()) } + /// Respond to GetNodeData request fn return_node_data(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { @@ -856,6 +893,7 @@ impl ChainSync { Ok(()) } + /// Respond to GetReceipts request fn return_receipts(&self, io: &mut SyncIo, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { let mut count = r.item_count(); if count == 0 { @@ -881,6 +919,7 @@ impl ChainSync { Ok(()) } + /// Dispatch incoming requests and responses pub fn on_packet(&mut self, io: &mut SyncIo, peer: &PeerId, packet_id: u8, data: &[u8]) { let rlp = UntrustedRlp::new(data); let result = match packet_id { @@ -904,6 +943,7 @@ impl ChainSync { }) } + /// Maintain other peers. Send out any new blocks and transactions pub fn maintain_sync(&mut self, _io: &mut SyncIo) { } } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index fc3553180..18f2288b1 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,88 +1,73 @@ +/// Blockchain sync module +/// Implements ethereum protocol version 63 as specified here: +/// https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol +/// +/// Usage example: +/// +/// ```rust +/// extern crate ethcore_util as util; +/// extern crate ethcore; +/// use std::env; +/// use std::sync::Arc; +/// use util::network::NetworkService; +/// use ethcore::client::Client; +/// use ethcore::sync::EthSync; +/// use ethcore::spec::Spec; +/// +/// fn main() { +/// let mut service = NetworkService::start().unwrap(); +/// let frontier = Spec::new_frontier(); +/// let dir = env::temp_dir(); +/// let client = Arc::new(Client::new(&frontier.genesis_block(), &dir)); +/// EthSync::register(&mut service, client); +/// } +/// ``` + use std::sync::Arc; use client::BlockChainClient; -use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, PacketId, Message, Error as NetworkError}; +use util::network::{ProtocolHandler, NetworkService, HandlerIo, TimerToken, PeerId, Message}; use sync::chain::ChainSync; +use sync::io::NetSyncIo; mod chain; +mod io; mod range_collection; #[cfg(test)] mod tests; -pub fn new(_service: &mut NetworkService, eth_client: Arc) -> EthSync { - EthSync { - chain: eth_client, - sync: ChainSync::new(), - } -} - -pub trait SyncIo { - fn disable_peer(&mut self, peer_id: &PeerId); - fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; - fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; - fn chain<'s>(&'s mut self) -> &'s mut BlockChainClient; -} - -pub struct NetSyncIo<'s, 'h> where 'h:'s { - network: &'s mut HandlerIo<'h>, - chain: &'s mut BlockChainClient -} - -impl<'s, 'h> NetSyncIo<'s, 'h> { - pub fn new(network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient) -> NetSyncIo<'s,'h> { - NetSyncIo { - network: network, - chain: chain, - } - } -} - -impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { - fn disable_peer(&mut self, peer_id: &PeerId) { - self.network.disable_peer(*peer_id); - } - - fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ - self.network.respond(packet_id, data) - } - - fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ - self.network.send(peer_id, packet_id, data) - } - - fn chain<'a>(&'a mut self) -> &'a mut BlockChainClient { - self.chain - } -} - +/// Ethereum network protocol handler pub struct EthSync { + /// Shared blockchain client. TODO: this should evetually become an IPC endpoint chain: Arc, + /// Sync strategy sync: ChainSync } pub use self::chain::SyncStatus; impl EthSync { - pub fn new(chain: Arc) -> EthSync { - EthSync { + /// Creates and register protocol with the network service + pub fn register(service: &mut NetworkService, chain: Arc) { + let sync = Box::new(EthSync { chain: chain, sync: ChainSync::new(), - } - } - - pub fn is_syncing(&self) -> bool { - self.sync.is_syncing() + }); + service.register_protocol(sync, "eth", &[62u8, 63u8]).expect("Error registering eth protocol handler"); } + /// Get sync status pub fn status(&self) -> SyncStatus { self.sync.status() } - pub fn stop_network(&mut self, io: &mut HandlerIo) { + /// Stop sync + pub fn stop(&mut self, io: &mut HandlerIo) { self.sync.abort(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } - pub fn start_network(&mut self, io: &mut HandlerIo) { + /// Restart sync + pub fn restart(&mut self, io: &mut HandlerIo) { self.sync.restart(&mut NetSyncIo::new(io, Arc::get_mut(&mut self.chain).unwrap())); } } diff --git a/src/sync/range_collection.rs b/src/sync/range_collection.rs index 822e88d75..d212625be 100644 --- a/src/sync/range_collection.rs +++ b/src/sync/range_collection.rs @@ -1,3 +1,6 @@ +/// This module defines a trait for a collection of ranged values and an implementation +/// for this trait over sorted vector. + use std::ops::{Add, Sub, Range}; pub trait ToUsize { @@ -8,16 +11,28 @@ pub trait FromUsize { fn from_usize(s: usize) -> Self; } +/// A key-value collection orderd by key with sequential key-value pairs grouped together. +/// Such group is called a range. +/// E.g. a set of collection of 5 pairs {1, a}, {2, b}, {10, x}, {11, y}, {12, z} will be grouped into two ranges: {1, [a,b]}, {10, [x,y,z]} pub trait RangeCollection { + /// Check if the given key is present in the collection. fn have_item(&self, key: &K) -> bool; + /// Get value by key. fn find_item(&self, key: &K) -> Option<&V>; + /// Get a range of keys from `key` till the end of the range that has `key` + /// Returns an empty range is key does not exist. fn get_tail(&mut self, key: &K) -> Range; + /// Remove all elements < `start` in the range that contains `start` - 1 fn remove_head(&mut self, start: &K); + /// Remove all elements >= `start` in the range that contains `start` fn remove_tail(&mut self, start: &K); + /// Remove all elements >= `tail` fn insert_item(&mut self, key: K, value: V); + /// Get an iterator over ranges fn range_iter<'c>(&'c self) -> RangeIterator<'c, K, V>; } +/// Range iterator. For each range yelds a key for the first element of the range and a vector of values. pub struct RangeIterator<'c, K:'c, V:'c> { range: usize, collection: &'c Vec<(K, Vec)> @@ -72,7 +87,6 @@ impl RangeCollection for Vec<(K, Vec)> where K: Ord + PartialEq + } } - /// Get a range of elements from start till the end of the range fn get_tail(&mut self, key: &K) -> Range { let kv = *key; match self.binary_search_by(|&(k, _)| k.cmp(key).reverse()) { diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 79e1443c9..28e526aa9 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -7,8 +7,8 @@ use util::rlp::{self, Rlp, RlpStream, View, Stream}; use util::network::{PeerId, PacketId, Error as NetworkError}; use client::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; use header::Header as BlockHeader; -use sync::{SyncIo}; -use sync::chain::{ChainSync}; +use sync::io::SyncIo; +use sync::chain::ChainSync; struct TestBlockChainClient { blocks: HashMap, From 0eb69c7f1c9c592cdb6997356e41957dd64bb105 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 19:10:58 +0100 Subject: [PATCH 20/27] Removed verification from this branch --- src/verification.rs | 109 -------------------------------------------- 1 file changed, 109 deletions(-) delete mode 100644 src/verification.rs diff --git a/src/verification.rs b/src/verification.rs deleted file mode 100644 index 95951da53..000000000 --- a/src/verification.rs +++ /dev/null @@ -1,109 +0,0 @@ -use util::uint::*; -use util::hash::*; -use util::rlp::*; -use util::sha3::Hashable; -use util::triehash::ordered_trie_root; -use header::Header; -use client::BlockNumber; - -#[derive(Debug)] -pub enum TransactionVerificationError { - OutOfGasBase, - OutOfGasIntrinsic, - NotEnoughCash, - GasPriceTooLow, - BlockGasLimitReached, - FeeTooSmall, - TooMuchGasUsed { - used: U256, - limit: U256 - }, - ExtraDataTooBig, - InvalidSignature, - InvalidTransactionFormat, -} - -#[derive(Debug)] -pub enum BlockVerificationError { - TooMuchGasUsed { - used: U256, - limit: U256, - }, - InvalidBlockFormat, - InvalidUnclesHash { - expected: H256, - got: H256, - }, - TooManyUncles, - UncleTooOld, - UncleIsBrother, - UncleInChain, - UncleParentNotInChain, - InvalidStateRoot, - InvalidGasUsed, - InvalidTransactionsRoot { - expected: H256, - got: H256, - }, - InvalidDifficulty, - InvalidGasLimit, - InvalidReceiptsStateRoot, - InvalidTimestamp, - InvalidLogBloom, - InvalidNonce, - InvalidBlockHeaderItemCount, - InvalidBlockNonce, - InvalidParentHash, - InvalidUncleParentHash, - InvalidNumber, - BlockNotFound, - UnknownParent, -} - - -pub fn verify_header(header: &Header) -> Result<(), BlockVerificationError> { - if header.number > From::from(BlockNumber::max_value()) { - return Err(BlockVerificationError::InvalidNumber) - } - if header.gas_used > header.gas_limit { - return Err(BlockVerificationError::TooMuchGasUsed { - used: header.gas_used, - limit: header.gas_limit, - }); - } - Ok(()) -} - -pub fn verify_parent(header: &Header, parent: &Header) -> Result<(), BlockVerificationError> { - if !header.parent_hash.is_zero() && parent.hash() != header.parent_hash { - return Err(BlockVerificationError::InvalidParentHash) - } - if header.timestamp <= parent.timestamp { - return Err(BlockVerificationError::InvalidTimestamp) - } - if header.number <= parent.number { - return Err(BlockVerificationError::InvalidNumber) - } - Ok(()) -} - -pub fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), BlockVerificationError> { - let block = Rlp::new(block); - let tx = block.at(1); - let expected_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here - if &expected_root != transactions_root { - return Err(BlockVerificationError::InvalidTransactionsRoot { - expected: expected_root.clone(), - got: transactions_root.clone(), - }); - } - let expected_uncles = block.at(2).as_raw().sha3(); - if &expected_uncles != uncles_hash { - return Err(BlockVerificationError::InvalidUnclesHash { - expected: expected_uncles.clone(), - got: uncles_hash.clone(), - }); - } - Ok(()) -} - From 32bfa69106a4c370fe8a2e6e464c6159c9414bb9 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 19:13:58 +0100 Subject: [PATCH 21/27] More docs --- src/sync/chain.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index b7d550c9b..276b1993c 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -164,6 +164,7 @@ pub struct ChainSync { impl ChainSync { + /// Create a new instance of syncing strategy. pub fn new() -> ChainSync { ChainSync { state: SyncState::NotSynced, From 85ddbba893c41e8ee6ff714ae80ac01c011083e4 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 23:21:57 +0100 Subject: [PATCH 22/27] Enabled logger for client app --- src/bin/client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/client.rs b/src/bin/client.rs index 2b003dca9..09e414476 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -1,6 +1,7 @@ extern crate ethcore_util as util; extern crate ethcore; extern crate rustc_serialize; +extern crate env_logger; use std::io::*; use std::env; @@ -12,6 +13,7 @@ use ethcore::sync::EthSync; use ethcore::ethereum; fn main() { + ::env_logger::init().ok(); let mut service = NetworkService::start().unwrap(); //TODO: replace with proper genesis and chain params. let frontier = ethereum::new_frontier(); From 51584f520244ac50a53f0708394abc5270fa886c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 14:11:23 +0100 Subject: [PATCH 23/27] Networking fixes --- src/sync/chain.rs | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 276b1993c..95505ff81 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -13,20 +13,11 @@ /// 4. Maintain sync by handling NewBlocks/NewHashes messages /// -use std::collections::{HashSet, HashMap}; -use std::cmp::{min, max}; +use util::*; use std::mem::{replace}; -use util::network::{PeerId, PacketId}; -use util::hash::{H256, FixedHash}; -use util::bytes::{Bytes}; -use util::uint::{U256}; -use util::rlp::{Rlp, UntrustedRlp, RlpStream, self}; -use util::rlp::rlptraits::{Stream, View}; -use util::rlp::rlperrors::DecoderError; -use util::sha3::Hashable; -use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; use views::{HeaderView}; use header::{Header as BlockHeader}; +use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; use sync::io::SyncIo; @@ -66,6 +57,8 @@ const NODE_DATA_PACKET: u8 = 0x0e; const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; +const NETWORK_ID: U256 = ONE_U256; //TODO: get this from parent + struct Header { /// Header data data: Bytes, @@ -241,7 +234,19 @@ impl ChainSync { asking_blocks: Vec::new(), }; - trace!(target: "sync", "New peer (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer.protocol_version, peer.network_id, peer.difficulty, peer.latest, peer.genesis); + trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest, peer.genesis); + + let chain_info = io.chain().chain_info(); + if peer.genesis != chain_info.genesis_hash { + io.disable_peer(peer_id); + trace!(target: "sync", "Peer {} genesis hash not matched", peer_id); + return Ok(()); + } + if peer.network_id != NETWORK_ID { + io.disable_peer(peer_id); + trace!(target: "sync", "Peer {} network id not matched", peer_id); + return Ok(()); + } let old = self.peers.insert(peer_id.clone(), peer); if old.is_some() { @@ -449,8 +454,10 @@ impl ChainSync { /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: &PeerId) { trace!(target: "sync", "== Disconnected {}", peer); - self.clear_peer_download(peer); - self.continue_sync(io); + if self.peers.contains_key(&peer) { + self.clear_peer_download(peer); + self.continue_sync(io); + } } /// Called when a new peer is connected @@ -769,7 +776,7 @@ impl ChainSync { let mut packet = RlpStream::new_list(5); let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); - packet.append(&0u32); //TODO: network id + packet.append(&NETWORK_ID); //TODO: network id packet.append(&chain.total_difficulty); packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); From 5f5f26de48f8566237b0bab911f8ecdf25711bb4 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 15:08:57 +0100 Subject: [PATCH 24/27] Do not insert new blocks out of order --- src/sync/chain.rs | 52 +++++++++++++++++++++++++---------------------- src/sync/io.rs | 11 +++++----- src/sync/tests.rs | 7 ++++--- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/sync/chain.rs b/src/sync/chain.rs index 95505ff81..bfbf3cfb8 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -386,31 +386,35 @@ impl ChainSync { let h = header_rlp.as_raw().sha3(); trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h); - match io.chain().import_block(block_rlp.as_raw()) { - ImportResult::AlreadyInChain => { - trace!(target: "sync", "New block already in chain {:?}", h); - }, - ImportResult::AlreadyQueued(_) => { - trace!(target: "sync", "New block already queued {:?}", h); - }, - ImportResult::Queued(QueueStatus::Known) => { - trace!(target: "sync", "New block queued {:?}", h); - }, - ImportResult::Queued(QueueStatus::Unknown) => { - trace!(target: "sync", "New block unknown {:?}", h); - //TODO: handle too many unknown blocks - let difficulty: U256 = try!(r.val_at(1)); - let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; - if difficulty > peer_difficulty { - trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); - self.sync_peer(io, peer_id, true); + let header_view = HeaderView::new(header_rlp.as_raw()); + // TODO: Decompose block and add to self.headers and self.bodies instead + if header_view.number() == From::from(self.last_imported_block + 1) { + match io.chain().import_block(block_rlp.as_raw()) { + ImportResult::AlreadyInChain => { + trace!(target: "sync", "New block already in chain {:?}", h); + }, + ImportResult::AlreadyQueued(_) => { + trace!(target: "sync", "New block already queued {:?}", h); + }, + ImportResult::Queued(QueueStatus::Known) => { + trace!(target: "sync", "New block queued {:?}", h); + }, + ImportResult::Queued(QueueStatus::Unknown) => { + trace!(target: "sync", "New block unknown {:?}", h); + //TODO: handle too many unknown blocks + let difficulty: U256 = try!(r.val_at(1)); + let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; + if difficulty > peer_difficulty { + trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); + self.sync_peer(io, peer_id, true); + } + }, + ImportResult::Bad =>{ + debug!(target: "sync", "Bad new block {:?}", h); + io.disable_peer(peer_id); } - }, - ImportResult::Bad =>{ - debug!(target: "sync", "Bad new block {:?}", h); - io.disable_peer(peer_id); - } - }; + }; + } Ok(()) } diff --git a/src/sync/io.rs b/src/sync/io.rs index 54bd22f14..ed7b0fec5 100644 --- a/src/sync/io.rs +++ b/src/sync/io.rs @@ -1,10 +1,11 @@ use client::BlockChainClient; -use util::network::{HandlerIo, PeerId, PacketId, Error as NetworkError}; +use util::network::{HandlerIo, PeerId, PacketId,}; +use util::error::UtilError; pub trait SyncIo { fn disable_peer(&mut self, peer_id: &PeerId); - fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; - fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; + fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError>; + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError>; fn chain<'s>(&'s mut self) -> &'s mut BlockChainClient; } @@ -27,11 +28,11 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { self.network.disable_peer(*peer_id); } - fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ + fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError>{ self.network.respond(packet_id, data) } - fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>{ + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError>{ self.network.send(peer_id, packet_id, data) } diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 28e526aa9..dfcf75c8b 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -4,7 +4,8 @@ use util::hash::{H256, FixedHash}; use util::uint::{U256}; use util::sha3::Hashable; use util::rlp::{self, Rlp, RlpStream, View, Stream}; -use util::network::{PeerId, PacketId, Error as NetworkError}; +use util::network::{PeerId, PacketId}; +use util::error::UtilError; use client::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; use header::Header as BlockHeader; use sync::io::SyncIo; @@ -195,7 +196,7 @@ impl<'p> SyncIo for TestIo<'p> { fn disable_peer(&mut self, _peer_id: &PeerId) { } - fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), NetworkError> { + fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { self.queue.push_back(TestPacket { data: data, packet_id: packet_id, @@ -204,7 +205,7 @@ impl<'p> SyncIo for TestIo<'p> { Ok(()) } - fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError> { + fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { self.queue.push_back(TestPacket { data: data, packet_id: packet_id, From 452294ab8dc2882be0773b01f4355992f63a4682 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 23:37:09 +0100 Subject: [PATCH 25/27] ImportResult becomes a result --- src/blockchain.rs | 35 ++++++++++++++----------- src/client.rs | 30 +++++---------------- src/error.rs | 7 +++++ src/queue.rs | 22 +++++++--------- src/sync/chain.rs | 66 ++++++++++++++++++++++++++--------------------- src/sync/io.rs | 9 +++++++ src/sync/tests.rs | 12 ++++----- 7 files changed, 96 insertions(+), 85 deletions(-) diff --git a/src/blockchain.rs b/src/blockchain.rs index 3a1d24725..c6539e423 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -200,11 +200,16 @@ impl BlockChain { /// ```json /// { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 } /// ``` - pub fn tree_route(&self, from: H256, to: H256) -> TreeRoute { - let from_details = self.block_details(&from).expect("from hash is invalid!"); - let to_details = self.block_details(&to).expect("to hash is invalid!"); - - self._tree_route((from_details, from), (to_details, to)) + pub fn tree_route(&self, from: H256, to: H256) -> Option { + let from_details = match self.block_details(&from) { + Some(h) => h, + None => return None, + }; + let to_details = match self.block_details(&to) { + Some(h) => h, + None => return None, + }; + Some(self._tree_route((from_details, from), (to_details, to))) } /// Similar to `tree_route` function, but can be used to return a route @@ -597,52 +602,52 @@ mod tests { assert_eq!(bc.block_hash(&U256::from(3)).unwrap(), b3a_hash); // test trie route - let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()); + let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()).unwrap(); assert_eq!(r0_1.ancestor, genesis_hash); assert_eq!(r0_1.blocks, [b1_hash.clone()]); assert_eq!(r0_1.index, 0); - let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); + let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()).unwrap(); assert_eq!(r0_2.ancestor, genesis_hash); assert_eq!(r0_2.blocks, [b1_hash.clone(), b2_hash.clone()]); assert_eq!(r0_2.index, 0); - let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); + let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()).unwrap(); assert_eq!(r1_3a.ancestor, b1_hash); assert_eq!(r1_3a.blocks, [b2_hash.clone(), b3a_hash.clone()]); assert_eq!(r1_3a.index, 0); - let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); + let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()).unwrap(); assert_eq!(r1_3b.ancestor, b1_hash); assert_eq!(r1_3b.blocks, [b2_hash.clone(), b3b_hash.clone()]); assert_eq!(r1_3b.index, 0); - let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); + let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()).unwrap(); assert_eq!(r3a_3b.ancestor, b2_hash); assert_eq!(r3a_3b.blocks, [b3a_hash.clone(), b3b_hash.clone()]); assert_eq!(r3a_3b.index, 1); - let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()); + let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()).unwrap(); assert_eq!(r1_0.ancestor, genesis_hash); assert_eq!(r1_0.blocks, [b1_hash.clone()]); assert_eq!(r1_0.index, 1); - let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()); + let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()).unwrap(); assert_eq!(r2_0.ancestor, genesis_hash); assert_eq!(r2_0.blocks, [b2_hash.clone(), b1_hash.clone()]); assert_eq!(r2_0.index, 2); - let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()); + let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()).unwrap(); assert_eq!(r3a_1.ancestor, b1_hash); assert_eq!(r3a_1.blocks, [b3a_hash.clone(), b2_hash.clone()]); assert_eq!(r3a_1.index, 2); - let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()); + let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()).unwrap(); assert_eq!(r3b_1.ancestor, b1_hash); assert_eq!(r3b_1.blocks, [b3b_hash.clone(), b2_hash.clone()]); assert_eq!(r3b_1.index, 2); - let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()); + let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()).unwrap(); assert_eq!(r3b_3a.ancestor, b2_hash); assert_eq!(r3b_3a.blocks, [b3b_hash.clone(), b3a_hash.clone()]); assert_eq!(r3b_3a.index, 1); diff --git a/src/client.rs b/src/client.rs index e9d0b1485..88c828af8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,21 +2,14 @@ use std::sync::Arc; use util::*; use blockchain::BlockChain; use views::BlockView; - -/// Status for a block in a queue. -pub enum QueueStatus { - /// Part of the known chain. - Known, - /// Part of the unknown chain. - Unknown, -} +use error::ImportError; /// General block status pub enum BlockStatus { /// Part of the blockchain. InChain, /// Queued for import. - Queued(QueueStatus), + Queued, /// Known as bad. Bad, /// Unknown. @@ -24,16 +17,7 @@ pub enum BlockStatus { } /// Result of import block operation. -pub enum ImportResult { - /// Added to import queue. - Queued(QueueStatus), - /// Already in the chain. - AlreadyInChain, - /// Already queued for import. - AlreadyQueued(QueueStatus), - /// Bad or already known as bad. - Bad, -} +pub type ImportResult = Result<(), ImportError>; /// Information about the blockchain gthered together. pub struct BlockChainInfo { @@ -88,7 +72,7 @@ pub trait BlockChainClient : Sync { /// Get a tree route between `from` and `to`. /// See `BlockChain::tree_route`. - fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute; + fn tree_route(&self, from: &H256, to: &H256) -> Option; /// Get latest state node fn state_data(&self, hash: &H256) -> Option; @@ -165,7 +149,7 @@ impl BlockChainClient for Client { } } - fn tree_route(&self, from: &H256, to: &H256) -> TreeRoute { + fn tree_route(&self, from: &H256, to: &H256) -> Option { self.chain.tree_route(from.clone(), to.clone()) } @@ -184,11 +168,11 @@ impl BlockChainClient for Client { let header = block.header_view(); let hash = header.sha3(); if self.chain.is_known(&hash) { - return ImportResult::Bad; + return Err(ImportError::AlreadyInChain); } } self.chain.insert_block(bytes); - ImportResult::Queued(QueueStatus::Known) + Ok(()) } fn queue_status(&self) -> BlockQueueStatus { diff --git a/src/error.rs b/src/error.rs index 68b75ab5b..d975f15b6 100644 --- a/src/error.rs +++ b/src/error.rs @@ -23,6 +23,13 @@ pub enum BlockError { InvalidSealArity(Mismatch), } +#[derive(Debug)] +pub enum ImportError { + Bad(BlockError), + AlreadyInChain, + AlreadyQueued, +} + #[derive(Debug)] /// General error type which should be capable of representing all errors in ethcore. pub enum Error { diff --git a/src/queue.rs b/src/queue.rs index 8d427e8a1..ea212aaf1 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -1,26 +1,24 @@ use std::sync::Arc; -//use util::bytes::*; -use util::sha3::*; +use util::*; use blockchain::BlockChain; use client::{QueueStatus, ImportResult}; use views::{BlockView}; - -pub struct BlockQueue { - chain: Arc -} +/// A queue of blocks. Sits between network or other I/O and the BlockChain. +/// Sorts them ready for blockchain insertion. +pub struct BlockQueue; impl BlockQueue { - pub fn new(chain: Arc) -> BlockQueue { - BlockQueue { - chain: chain - } + /// Creates a new queue instance. + pub fn new() -> BlockQueue { } + /// Clear the queue and stop verification activity. pub fn clear(&mut self) { } - pub fn import_block(&mut self, bytes: &[u8]) -> ImportResult { + /// Add a block to the queue. + pub fn import_block(&mut self, bytes: &[u8], bc: &mut BlockChain) -> ImportResult { //TODO: verify block { let block = BlockView::new(bytes); @@ -30,7 +28,7 @@ impl BlockQueue { return ImportResult::Bad; } } - self.chain.insert_block(bytes); + bc.insert_block(bytes); ImportResult::Queued(QueueStatus::Known) } } diff --git a/src/sync/chain.rs b/src/sync/chain.rs index bfbf3cfb8..8ef9649b4 100644 --- a/src/sync/chain.rs +++ b/src/sync/chain.rs @@ -17,8 +17,9 @@ use util::*; use std::mem::{replace}; use views::{HeaderView}; use header::{Header as BlockHeader}; -use client::{BlockNumber, BlockChainClient, BlockStatus, QueueStatus, ImportResult}; +use client::{BlockNumber, BlockChainClient, BlockStatus}; use sync::range_collection::{RangeCollection, ToUsize, FromUsize}; +use error::*; use sync::io::SyncIo; impl ToUsize for BlockNumber { @@ -76,12 +77,13 @@ struct HeaderId { } #[derive(Copy, Clone, Eq, PartialEq, Debug)] +/// Sync state pub enum SyncState { /// Initial chain sync has not started yet NotSynced, /// Initial chain sync complete. Waiting for new packets Idle, - /// Block downloading paused. Waiting for block queue to process blocks and free space + /// Block downloading paused. Waiting for block queue to process blocks and free some space Waiting, /// Downloading blocks Blocks, @@ -108,24 +110,33 @@ pub struct SyncStatus { } #[derive(PartialEq, Eq, Debug)] +/// Peer data type requested enum PeerAsking { Nothing, BlockHeaders, BlockBodies, } +/// Syncing peer information struct PeerInfo { + /// eth protocol version protocol_version: u32, + /// Peer chain genesis hash genesis: H256, + /// Peer network id network_id: U256, + /// Peer best block hash latest: H256, + /// Peer total difficulty difficulty: U256, + /// Type of data currenty being requested from peer. asking: PeerAsking, + /// A set of block numbers being requested asking_blocks: Vec, } -type Body = Bytes; - +/// Blockchain sync handler. +/// See module documentation for more details. pub struct ChainSync { /// Sync state state: SyncState, @@ -140,7 +151,7 @@ pub struct ChainSync { /// Downloaded headers. headers: Vec<(BlockNumber, Vec
)>, //TODO: use BTreeMap once range API is sable. For now it is a vector sorted in descending order /// Downloaded bodies - bodies: Vec<(BlockNumber, Vec)>, //TODO: use BTreeMap once range API is sable. For now it is a vector sorted in descending order + bodies: Vec<(BlockNumber, Vec)>, //TODO: use BTreeMap once range API is sable. For now it is a vector sorted in descending order /// Peer info peers: HashMap, /// Used to map body to header @@ -390,31 +401,31 @@ impl ChainSync { // TODO: Decompose block and add to self.headers and self.bodies instead if header_view.number() == From::from(self.last_imported_block + 1) { match io.chain().import_block(block_rlp.as_raw()) { - ImportResult::AlreadyInChain => { + Err(ImportError::AlreadyInChain) => { trace!(target: "sync", "New block already in chain {:?}", h); }, - ImportResult::AlreadyQueued(_) => { + Err(ImportError::AlreadyQueued) => { trace!(target: "sync", "New block already queued {:?}", h); }, - ImportResult::Queued(QueueStatus::Known) => { + Ok(()) => { trace!(target: "sync", "New block queued {:?}", h); }, - ImportResult::Queued(QueueStatus::Unknown) => { - trace!(target: "sync", "New block unknown {:?}", h); - //TODO: handle too many unknown blocks - let difficulty: U256 = try!(r.val_at(1)); - let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; - if difficulty > peer_difficulty { - trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); - self.sync_peer(io, peer_id, true); - } - }, - ImportResult::Bad =>{ - debug!(target: "sync", "Bad new block {:?}", h); + Err(e) => { + debug!(target: "sync", "Bad new block {:?} : {:?}", h, e); io.disable_peer(peer_id); } }; } + else { + trace!(target: "sync", "New block unknown {:?}", h); + //TODO: handle too many unknown blocks + let difficulty: U256 = try!(r.val_at(1)); + let peer_difficulty = self.peers.get_mut(&peer_id).expect("ChainSync: unknown peer").difficulty; + if difficulty > peer_difficulty { + trace!(target: "sync", "Received block {:?} with no known parent. Peer needs syncing...", h); + self.sync_peer(io, peer_id, true); + } + } Ok(()) } @@ -434,7 +445,7 @@ impl ChainSync { BlockStatus::InChain => { trace!(target: "sync", "New block hash already in chain {:?}", h); }, - BlockStatus::Queued(_) => { + BlockStatus::Queued => { trace!(target: "sync", "New hash block already queued {:?}", h); }, BlockStatus::Unknown => { @@ -642,27 +653,24 @@ impl ChainSync { block_rlp.append_raw(body.at(1).as_raw(), 1); let h = &headers.1[i].hash; match io.chain().import_block(&block_rlp.out()) { - ImportResult::AlreadyInChain => { + Err(ImportError::AlreadyInChain) => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; self.last_imported_hash = h.clone(); }, - ImportResult::AlreadyQueued(_) => { + Err(ImportError::AlreadyQueued) => { trace!(target: "sync", "Block already queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; self.last_imported_hash = h.clone(); }, - ImportResult::Queued(QueueStatus::Known) => { + Ok(()) => { trace!(target: "sync", "Block queued {:?}", h); self.last_imported_block = headers.0 + i as BlockNumber; self.last_imported_hash = h.clone(); imported += 1; }, - ImportResult::Queued(QueueStatus::Unknown) => { - panic!("Queued out of order block"); - }, - ImportResult::Bad =>{ - debug!(target: "sync", "Bad block {:?}", h); + Err(e) => { + debug!(target: "sync", "Bad block {:?} : {:?}", h, e); restart = true; } } diff --git a/src/sync/io.rs b/src/sync/io.rs index ed7b0fec5..9806a3bf5 100644 --- a/src/sync/io.rs +++ b/src/sync/io.rs @@ -2,19 +2,28 @@ use client::BlockChainClient; use util::network::{HandlerIo, PeerId, PacketId,}; use util::error::UtilError; +/// IO interface for the syning handler. +/// Provides peer connection management and an interface to the blockchain client. +// TODO: ratings pub trait SyncIo { + /// Disable a peer fn disable_peer(&mut self, peer_id: &PeerId); + /// Respond to current request with a packet. Can be called from an IO handler for incoming packet. fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError>; + /// Send a packet to a peer. fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError>; + /// Get the blockchain fn chain<'s>(&'s mut self) -> &'s mut BlockChainClient; } +/// Wraps `HandlerIo` and the blockchain client pub struct NetSyncIo<'s, 'h> where 'h:'s { network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient } impl<'s, 'h> NetSyncIo<'s, 'h> { + /// Creates a new instance from the `HandlerIo` and the blockchain client reference. pub fn new(network: &'s mut HandlerIo<'h>, chain: &'s mut BlockChainClient) -> NetSyncIo<'s,'h> { NetSyncIo { network: network, diff --git a/src/sync/tests.rs b/src/sync/tests.rs index dfcf75c8b..b284a568e 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -6,7 +6,7 @@ use util::sha3::Hashable; use util::rlp::{self, Rlp, RlpStream, View, Stream}; use util::network::{PeerId, PacketId}; use util::error::UtilError; -use client::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult, QueueStatus}; +use client::{BlockChainClient, BlockStatus, BlockNumber, TreeRoute, BlockQueueStatus, BlockChainInfo, ImportResult}; use header::Header as BlockHeader; use sync::io::SyncIo; use sync::chain::ChainSync; @@ -49,7 +49,7 @@ impl TestBlockChainClient { rlp.append(&header); rlp.append_raw(&rlp::NULL_RLP, 1); rlp.append_raw(uncles.as_raw(), 1); - self.import_block(rlp.as_raw()); + self.import_block(rlp.as_raw()).unwrap(); } } } @@ -100,12 +100,12 @@ impl BlockChainClient for TestBlockChainClient { } } - fn tree_route(&self, _from: &H256, _to: &H256) -> TreeRoute { - TreeRoute { + fn tree_route(&self, _from: &H256, _to: &H256) -> Option { + Some(TreeRoute { blocks: Vec::new(), ancestor: H256::new(), index: 0 - } + }) } fn state_data(&self, _h: &H256) -> Option { @@ -153,7 +153,7 @@ impl BlockChainClient for TestBlockChainClient { else { self.blocks.insert(header.hash(), b.to_vec()); } - ImportResult::Queued(QueueStatus::Known) + Ok(()) } fn queue_status(&self) -> BlockQueueStatus { From 33d3a4d6339d258e753fcf9c2a2d0a7163e8b7b1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 11 Jan 2016 11:51:31 +0100 Subject: [PATCH 26/27] Engine and Spec are now thread safe --- | 0 src/bin/client.rs | 5 +++-- src/blockchain.rs | 1 - src/builtin.rs | 5 +++++ src/client.rs | 8 +++++--- src/engine.rs | 2 +- src/ethereum/ethash.rs | 4 ++++ src/ethereum/mod.rs | 6 +++--- src/spec.rs | 41 +++++++++++++++++++++++++++++++---------- src/sync/mod.rs | 5 +++-- 10 files changed, 55 insertions(+), 22 deletions(-) create mode 100644 diff --git a/ b/ new file mode 100644 index 000000000..e69de29bb diff --git a/src/bin/client.rs b/src/bin/client.rs index 09e414476..f8b29ac1c 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -11,15 +11,16 @@ use util::network::{NetworkService}; use ethcore::client::Client; use ethcore::sync::EthSync; use ethcore::ethereum; +use ethcore::ethereum::ethash::Ethash; fn main() { ::env_logger::init().ok(); let mut service = NetworkService::start().unwrap(); //TODO: replace with proper genesis and chain params. - let frontier = ethereum::new_frontier(); + let engine = Ethash::new_arc(ethereum::new_frontier()); let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - let client = Arc::new(Client::new(&frontier.genesis_block(), &dir)); + let client = Arc::new(Client::new(engine, &dir)); EthSync::register(&mut service, client); loop { let mut cmd = String::new(); diff --git a/src/blockchain.rs b/src/blockchain.rs index 59a82858e..c102ba1b4 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -1,6 +1,5 @@ //! Fast access to blockchain data. -use std::sync::*; use util::*; use rocksdb::{DB, WriteBatch, Writable}; use header::*; diff --git a/src/builtin.rs b/src/builtin.rs index 309b8f3e5..0c1e60d5f 100644 --- a/src/builtin.rs +++ b/src/builtin.rs @@ -12,6 +12,11 @@ pub struct Builtin { pub execute: Box, } +// Rust does not mark closurer that do not capture as Sync +// We promise that all builtins are thread safe since they only operate on given input. +unsafe impl Sync for Builtin {} +unsafe impl Send for Builtin {} + impl fmt::Debug for Builtin { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "") diff --git a/src/client.rs b/src/client.rs index 7f92a9a4b..019bdf797 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; use util::*; use blockchain::BlockChain; use views::BlockView; use error::ImportError; use header::BlockNumber; +use engine::Engine; /// General block status pub enum BlockStatus { @@ -95,13 +95,15 @@ pub trait BlockChainClient : Sync { /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. pub struct Client { chain: Arc, + _engine: Arc, } impl Client { - pub fn new(genesis: &[u8], path: &Path) -> Client { - let chain = Arc::new(BlockChain::new(genesis, path)); + pub fn new(engine: Arc, path: &Path) -> Client { + let chain = Arc::new(BlockChain::new(&engine.spec().genesis_block(), path)); Client { chain: chain.clone(), + _engine: engine.clone(), } } } diff --git a/src/engine.rs b/src/engine.rs index bfaf46348..64f85e079 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -4,7 +4,7 @@ use spec::Spec; /// A consensus mechanism for the chain. Generally either proof-of-work or proof-of-stake-based. /// Provides hooks into each of the major parts of block import. -pub trait Engine { +pub trait Engine : Sync + Send { /// The name of this engine. fn name(&self) -> &str; /// The version of this engine. Should be of the form diff --git a/src/ethereum/ethash.rs b/src/ethereum/ethash.rs index 828441fd6..d260e21b4 100644 --- a/src/ethereum/ethash.rs +++ b/src/ethereum/ethash.rs @@ -13,6 +13,10 @@ impl Ethash { pub fn new_boxed(spec: Spec) -> Box { Box::new(Ethash{spec: spec}) } + + pub fn new_arc(spec: Spec) -> Arc { + Arc::new(Ethash{spec: spec}) + } } impl Engine for Ethash { diff --git a/src/ethereum/mod.rs b/src/ethereum/mod.rs index ed1950d64..b3efe4a3d 100644 --- a/src/ethereum/mod.rs +++ b/src/ethereum/mod.rs @@ -52,7 +52,7 @@ mod tests { fn morden() { let morden = new_morden(); - assert_eq!(*morden.state_root(), H256::from_str("f3f4696bbf3b3b07775128eb7a3763279a394e382130f27c21e70233e04946a9").unwrap()); + assert_eq!(morden.state_root(), H256::from_str("f3f4696bbf3b3b07775128eb7a3763279a394e382130f27c21e70233e04946a9").unwrap()); let genesis = morden.genesis_block(); assert_eq!(BlockView::new(&genesis).header_view().sha3(), H256::from_str("0cd786a2425d16f152c658316c423e6ce1181e15c3295826d7c9904cba9ce303").unwrap()); @@ -63,10 +63,10 @@ mod tests { fn frontier() { let frontier = new_frontier(); - assert_eq!(*frontier.state_root(), H256::from_str("d7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544").unwrap()); + assert_eq!(frontier.state_root(), H256::from_str("d7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544").unwrap()); let genesis = frontier.genesis_block(); assert_eq!(BlockView::new(&genesis).header_view().sha3(), H256::from_str("d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3").unwrap()); let _ = frontier.to_engine(); } -} \ No newline at end of file +} diff --git a/src/spec.rs b/src/spec.rs index 0d821a6ad..acdfd2ecf 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -40,6 +40,27 @@ fn json_to_rlp_map(json: &Json) -> HashMap { }) } +//TODO: add code and data +#[derive(Debug)] +/// Genesis account data. Does no thave a DB overlay cache +pub struct GenesisAccount { + // Balance of the account. + balance: U256, + // Nonce of the account. + nonce: U256, +} + +impl GenesisAccount { + pub fn rlp(&self) -> Bytes { + let mut stream = RlpStream::new_list(4); + stream.append(&self.nonce); + stream.append(&self.balance); + stream.append(&SHA3_NULL_RLP); + stream.append(&SHA3_EMPTY); + stream.out() + } +} + /// Parameters for a block chain; includes both those intrinsic to the design of the /// chain and those to be interpreted by the active chain engine. #[derive(Debug)] @@ -62,12 +83,12 @@ pub struct Spec { pub gas_used: U256, pub timestamp: u64, pub extra_data: Bytes, - pub genesis_state: HashMap, + pub genesis_state: HashMap, pub seal_fields: usize, pub seal_rlp: Bytes, // May be prepopulated if we know this in advance. - state_root_memo: RefCell>, + state_root_memo: RwLock>, } impl Spec { @@ -82,11 +103,11 @@ impl Spec { } /// Return the state root for the genesis state, memoising accordingly. - pub fn state_root(&self) -> Ref { - if self.state_root_memo.borrow().is_none() { - *self.state_root_memo.borrow_mut() = Some(sec_trie_root(self.genesis_state.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect())); + pub fn state_root(&self) -> H256 { + if self.state_root_memo.read().unwrap().is_none() { + *self.state_root_memo.write().unwrap() = Some(sec_trie_root(self.genesis_state.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect())); } - Ref::map(self.state_root_memo.borrow(), |x|x.as_ref().unwrap()) + self.state_root_memo.read().unwrap().as_ref().unwrap().clone() } pub fn genesis_header(&self) -> Header { @@ -113,7 +134,7 @@ impl Spec { let r = Rlp::new(&seal); (0..self.seal_fields).map(|i| r.at(i).as_raw().to_vec()).collect() }, - hash: RefCell::new(None) + hash: RefCell::new(None), } } @@ -149,7 +170,7 @@ impl Spec { // let nonce = if let Some(&Json::String(ref n)) = acc.find("nonce") {U256::from_dec_str(n).unwrap_or(U256::from(0))} else {U256::from(0)}; // TODO: handle code & data if they exist. if balance.is_some() || nonce.is_some() { - state.insert(addr, Account::new_basic(balance.unwrap_or(U256::from(0)), nonce.unwrap_or(U256::from(0)))); + state.insert(addr, GenesisAccount { balance: balance.unwrap_or(U256::from(0)), nonce: nonce.unwrap_or(U256::from(0)) }); } } } @@ -186,7 +207,7 @@ impl Spec { genesis_state: state, seal_fields: seal_fields, seal_rlp: seal_rlp, - state_root_memo: RefCell::new(genesis.find("stateRoot").and_then(|_| genesis["stateRoot"].as_string()).map(|s| H256::from_str(&s[2..]).unwrap())), + state_root_memo: RwLock::new(genesis.find("stateRoot").and_then(|_| genesis["stateRoot"].as_string()).map(|s| H256::from_str(&s[2..]).unwrap())), } } @@ -228,7 +249,7 @@ mod tests { fn test_chain() { let test_spec = Spec::new_test(); - assert_eq!(*test_spec.state_root(), H256::from_str("f3f4696bbf3b3b07775128eb7a3763279a394e382130f27c21e70233e04946a9").unwrap()); + assert_eq!(test_spec.state_root(), H256::from_str("f3f4696bbf3b3b07775128eb7a3763279a394e382130f27c21e70233e04946a9").unwrap()); let genesis = test_spec.genesis_block(); assert_eq!(BlockView::new(&genesis).header_view().sha3(), H256::from_str("0cd786a2425d16f152c658316c423e6ce1181e15c3295826d7c9904cba9ce303").unwrap()); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index a070b2d59..aca40ffba 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -13,12 +13,13 @@ /// use ethcore::client::Client; /// use ethcore::sync::EthSync; /// use ethcore::ethereum; +/// use ethcore::ethereum::ethash::Ethash; /// /// fn main() { /// let mut service = NetworkService::start().unwrap(); -/// let frontier = ethereum::new_frontier(); +/// let engine = Ethash::new_arc(ethereum::new_frontier()); /// let dir = env::temp_dir(); -/// let client = Arc::new(Client::new(&frontier.genesis_block(), &dir)); +/// let client = Arc::new(Client::new(engine, &dir)); /// EthSync::register(&mut service, client); /// } /// ``` From 3a2663ce932094e11f4fe3b08b83e2e375345aa5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 11 Jan 2016 12:28:59 +0100 Subject: [PATCH 27/27] Client now takes Spec instead of Engine --- src/bin/client.rs | 5 ++--- src/client.rs | 16 +++++++++------- src/ethereum/ethash.rs | 4 ---- src/sync/mod.rs | 4 +--- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/bin/client.rs b/src/bin/client.rs index f8b29ac1c..a644ecd1b 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -11,16 +11,15 @@ use util::network::{NetworkService}; use ethcore::client::Client; use ethcore::sync::EthSync; use ethcore::ethereum; -use ethcore::ethereum::ethash::Ethash; fn main() { ::env_logger::init().ok(); let mut service = NetworkService::start().unwrap(); //TODO: replace with proper genesis and chain params. - let engine = Ethash::new_arc(ethereum::new_frontier()); + let spec = ethereum::new_frontier(); let mut dir = env::temp_dir(); dir.push(H32::random().hex()); - let client = Arc::new(Client::new(engine, &dir)); + let client = Arc::new(Client::new(spec, &dir).unwrap()); EthSync::register(&mut service, client); loop { let mut cmd = String::new(); diff --git a/src/client.rs b/src/client.rs index 019bdf797..b914dba19 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,8 +1,9 @@ use util::*; use blockchain::BlockChain; use views::BlockView; -use error::ImportError; +use error::*; use header::BlockNumber; +use spec::Spec; use engine::Engine; /// General block status @@ -95,16 +96,17 @@ pub trait BlockChainClient : Sync { /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. pub struct Client { chain: Arc, - _engine: Arc, + _engine: Arc>, } impl Client { - pub fn new(engine: Arc, path: &Path) -> Client { - let chain = Arc::new(BlockChain::new(&engine.spec().genesis_block(), path)); - Client { + pub fn new(spec: Spec, path: &Path) -> Result { + let chain = Arc::new(BlockChain::new(&spec.genesis_block(), path)); + let engine = Arc::new(try!(spec.to_engine())); + Ok(Client { chain: chain.clone(), - _engine: engine.clone(), - } + _engine: engine, + }) } } diff --git a/src/ethereum/ethash.rs b/src/ethereum/ethash.rs index d260e21b4..828441fd6 100644 --- a/src/ethereum/ethash.rs +++ b/src/ethereum/ethash.rs @@ -13,10 +13,6 @@ impl Ethash { pub fn new_boxed(spec: Spec) -> Box { Box::new(Ethash{spec: spec}) } - - pub fn new_arc(spec: Spec) -> Arc { - Arc::new(Ethash{spec: spec}) - } } impl Engine for Ethash { diff --git a/src/sync/mod.rs b/src/sync/mod.rs index aca40ffba..300465014 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -13,13 +13,11 @@ /// use ethcore::client::Client; /// use ethcore::sync::EthSync; /// use ethcore::ethereum; -/// use ethcore::ethereum::ethash::Ethash; /// /// fn main() { /// let mut service = NetworkService::start().unwrap(); -/// let engine = Ethash::new_arc(ethereum::new_frontier()); /// let dir = env::temp_dir(); -/// let client = Arc::new(Client::new(engine, &dir)); +/// let client = Arc::new(Client::new(ethereum::new_frontier(), &dir).unwrap()); /// EthSync::register(&mut service, client); /// } /// ```