From 35cabecad885997ffa328b6b123de5a8ae12dfb8 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 23 Feb 2016 18:51:29 +0100 Subject: [PATCH 01/89] support for polling --- Cargo.lock | 9 ++ ethcore/src/filter.rs | 16 ++++ rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/helpers/mod.rs | 21 +++++ rpc/src/v1/helpers/poll_filter.rs | 10 ++ rpc/src/v1/helpers/poll_indexer.rs | 144 +++++++++++++++++++++++++++++ rpc/src/v1/impls/eth.rs | 73 +++++++++++++-- rpc/src/v1/mod.rs | 1 + 9 files changed, 267 insertions(+), 9 deletions(-) create mode 100644 rpc/src/v1/helpers/mod.rs create mode 100644 rpc/src/v1/helpers/poll_filter.rs create mode 100644 rpc/src/v1/helpers/poll_indexer.rs diff --git a/Cargo.lock b/Cargo.lock index 779220fb6..fe1122195 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -211,6 +211,7 @@ dependencies = [ "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", + "transient-hashmap 0.1.0 (git+https://github.com/debris/transient-hashmap)", ] [[package]] @@ -728,6 +729,14 @@ name = "traitobject" version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "transient-hashmap" +version = "0.1.0" +source = "git+https://github.com/debris/transient-hashmap#e21bf844277785504ddc30ee22d2a709103d4992" +dependencies = [ + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "typeable" version = "0.1.2" diff --git a/ethcore/src/filter.rs b/ethcore/src/filter.rs index f5f9135d6..73aea8ccc 100644 --- a/ethcore/src/filter.rs +++ b/ethcore/src/filter.rs @@ -42,6 +42,22 @@ pub struct Filter { pub topics: [Option>; 4], } +impl Clone for Filter { + fn clone(&self) -> Self { + let mut topics = [None, None, None, None]; + for i in 0..4 { + topics[i] = self.topics[i].clone(); + } + + Filter { + from_block: self.from_block.clone(), + to_block: self.to_block.clone(), + address: self.address.clone(), + topics: topics + } + } +} + impl Filter { /// Returns combinations of each address and topic. pub fn bloom_possibilities(&self) -> Vec { diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index e36048690..03cffbb85 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -19,6 +19,7 @@ ethsync = { path = "../sync" } clippy = { version = "0.0.44", optional = true } rustc-serialize = "0.3" serde_macros = { version = "0.6.13", optional = true } +transient-hashmap = { git = "https://github.com/debris/transient-hashmap" } [build-dependencies] serde_codegen = { version = "0.6.13", optional = true } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index d7b5bdc3b..70d6fae82 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -27,6 +27,7 @@ extern crate jsonrpc_http_server; extern crate ethcore_util as util; extern crate ethcore; extern crate ethsync; +extern crate transient_hashmap; #[cfg(feature = "serde_macros")] include!("lib.rs.in"); diff --git a/rpc/src/v1/helpers/mod.rs b/rpc/src/v1/helpers/mod.rs new file mode 100644 index 000000000..d212d74ab --- /dev/null +++ b/rpc/src/v1/helpers/mod.rs @@ -0,0 +1,21 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +mod poll_indexer; +mod poll_filter; + +pub use self::poll_indexer::PollIndexer; +pub use self::poll_filter::PollFilter; diff --git a/rpc/src/v1/helpers/poll_filter.rs b/rpc/src/v1/helpers/poll_filter.rs new file mode 100644 index 000000000..465290270 --- /dev/null +++ b/rpc/src/v1/helpers/poll_filter.rs @@ -0,0 +1,10 @@ +//! Helper type with all filter possibilities. + +use ethcore::filter::Filter; + +#[derive(Clone)] +pub enum PollFilter { + Block, + PendingTransaction, + Logs(Filter) +} diff --git a/rpc/src/v1/helpers/poll_indexer.rs b/rpc/src/v1/helpers/poll_indexer.rs new file mode 100644 index 000000000..11cb30935 --- /dev/null +++ b/rpc/src/v1/helpers/poll_indexer.rs @@ -0,0 +1,144 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Indexes all rpc poll requests. + +use transient_hashmap::{TransientHashMap, Timer, StandardTimer}; + +/// Lifetime of poll (in seconds). +const POLL_LIFETIME: u64 = 60; + +pub type PollId = usize; +pub type BlockNumber = u64; + +pub struct PollInfo { + pub filter: F, + pub block_number: BlockNumber +} + +impl Clone for PollInfo where F: Clone { + fn clone(&self) -> Self { + PollInfo { + filter: self.filter.clone(), + block_number: self.block_number.clone() + } + } +} + +/// Indexes all poll requests. +/// +/// Lazily garbage collects unused polls info. +pub struct PollIndexer where T: Timer { + polls: TransientHashMap, T>, + next_available_id: PollId +} + +impl PollIndexer { + /// Creates new instance of indexer. + pub fn new() -> Self { + PollIndexer::new_with_timer(Default::default()) + } +} + +impl PollIndexer where T: Timer { + pub fn new_with_timer(timer: T) -> Self { + PollIndexer { + polls: TransientHashMap::new_with_timer(POLL_LIFETIME, timer), + next_available_id: 0 + } + } + + /// Returns id which can be used for new poll. + /// + /// Stores information when last poll happend. + pub fn create_poll(&mut self, filter: F, block: BlockNumber) -> PollId { + self.polls.prune(); + let id = self.next_available_id; + self.next_available_id += 1; + self.polls.insert(id, PollInfo { + filter: filter, + block_number: block + }); + id + } + + /// Updates information when last poll happend. + pub fn update_poll(&mut self, id: &PollId, block: BlockNumber) { + self.polls.prune(); + if let Some(info) = self.polls.get_mut(id) { + info.block_number = block; + } + } + + /// Returns number of block when last poll happend. + pub fn get_poll(&mut self, id: &PollId) -> Option<&PollInfo> { + self.polls.prune(); + self.polls.get(id) + } + + /// Removes poll info. + pub fn remove_poll(&mut self, id: &PollId) { + self.polls.remove(id); + } +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use transient_hashmap::Timer; + use v1::helpers::PollIndexer; + + struct TestTimer<'a> { + time: &'a RefCell + } + + impl<'a> Timer for TestTimer<'a> { + fn get_time(&self) -> i64 { + *self.time.borrow() + } + } + + #[test] + fn test_poll_indexer() { + let time = RefCell::new(0); + let timer = TestTimer { + time: &time + }; + + let mut indexer = PollIndexer::new_with_timer(timer); + assert_eq!(indexer.create_poll(false, 20), 0); + assert_eq!(indexer.create_poll(true, 20), 1); + + *time.borrow_mut() = 10; + indexer.update_poll(&0, 21); + assert_eq!(indexer.get_poll(&0).unwrap().filter, false); + assert_eq!(indexer.get_poll(&0).unwrap().block_number, 21); + + *time.borrow_mut() = 30; + indexer.update_poll(&1, 23); + assert_eq!(indexer.get_poll(&1).unwrap().filter, true); + assert_eq!(indexer.get_poll(&1).unwrap().block_number, 23); + + *time.borrow_mut() = 75; + indexer.update_poll(&0, 30); + assert!(indexer.get_poll(&0).is_none()); + assert_eq!(indexer.get_poll(&1).unwrap().filter, true); + assert_eq!(indexer.get_poll(&1).unwrap().block_number, 23); + + indexer.remove_poll(&1); + assert!(indexer.get_poll(&1).is_none()); + } +} diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 689afd019..153a51216 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Eth rpc implementation. -use std::sync::Arc; +use std::sync::{Mutex, Arc}; use ethsync::{EthSync, SyncState}; use jsonrpc_core::*; use util::hash::*; @@ -26,6 +26,7 @@ use ethcore::views::*; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, OptionalValue, Index, Filter, Log}; +use v1::helpers::{PollFilter, PollIndexer}; /// Eth rpc implementation. pub struct EthClient { @@ -212,28 +213,82 @@ impl Eth for EthClient { /// Eth filter rpc implementation. pub struct EthFilterClient { - client: Arc + client: Arc, + polls: Mutex> } impl EthFilterClient { /// Creates new Eth filter client. pub fn new(client: Arc) -> Self { EthFilterClient { - client: client + client: client, + polls: Mutex::new(PollIndexer::new()) } } } impl EthFilter for EthFilterClient { - fn new_block_filter(&self, _params: Params) -> Result { - Ok(Value::U64(0)) + fn new_filter(&self, params: Params) -> Result { + from_params::<(Filter,)>(params) + .and_then(|(filter,)| { + let mut polls = self.polls.lock().unwrap(); + let id = polls.create_poll(PollFilter::Logs(filter.into()), self.client.chain_info().best_block_number); + to_value(&U256::from(id)) + }) } - fn new_pending_transaction_filter(&self, _params: Params) -> Result { - Ok(Value::U64(1)) + fn new_block_filter(&self, params: Params) -> Result { + match params { + Params::None => { + let mut polls = self.polls.lock().unwrap(); + let id = polls.create_poll(PollFilter::Block, self.client.chain_info().best_block_number); + to_value(&U256::from(id)) + }, + _ => Err(Error::invalid_params()) + } } - fn filter_changes(&self, _: Params) -> Result { - to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + fn new_pending_transaction_filter(&self, params: Params) -> Result { + match params { + Params::None => { + let mut polls = self.polls.lock().unwrap(); + let id = polls.create_poll(PollFilter::PendingTransaction, self.client.chain_info().best_block_number); + to_value(&U256::from(id)) + }, + _ => Err(Error::invalid_params()) + } + } + + fn filter_changes(&self, params: Params) -> Result { + from_params::<(Index,)>(params) + .and_then(|(index,)| { + let info = self.polls.lock().unwrap().get_poll(&index.value()).cloned(); + match info { + None => Ok(Value::Array(vec![] as Vec)), + Some(info) => match info.filter { + PollFilter::Block => { + //unimplemented!() + to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + }, + PollFilter::PendingTransaction => { + //unimplemented!() + to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + }, + PollFilter::Logs(mut filter) => { + filter.from_block = BlockId::Number(info.block_number); + filter.to_block = BlockId::Latest; + let logs = self.client.logs(filter) + .into_iter() + .map(From::from) + .collect::>(); + + let current_number = self.client.chain_info().best_block_number; + self.polls.lock().unwrap().update_poll(&index.value(), current_number); + + to_value(&logs) + } + } + } + }) } } diff --git a/rpc/src/v1/mod.rs b/rpc/src/v1/mod.rs index 01635e872..57f7a944a 100644 --- a/rpc/src/v1/mod.rs +++ b/rpc/src/v1/mod.rs @@ -23,6 +23,7 @@ mod impls; mod types; #[cfg(test)] mod tests; +mod helpers; pub use self::traits::{Web3, Eth, EthFilter, Net}; pub use self::impls::*; From 0318907fb39e6535a69524e0a6ec419063695c83 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 24 Feb 2016 14:16:05 +0100 Subject: [PATCH 02/89] rpc eth_getFilterChanges returns new blocks, implemented eth_uninstallFilter --- ethcore/src/client.rs | 8 ++++++++ rpc/src/v1/impls/eth.rs | 21 ++++++++++++++++++--- sync/src/tests/helpers.rs | 4 ++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 9244c26fc..3a1a80141 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -114,6 +114,9 @@ pub trait BlockChainClient : Sync + Send { /// Get block total difficulty. fn block_total_difficulty(&self, id: BlockId) -> Option; + /// Get block hash. + fn block_hash(&self, id: BlockId) -> Option; + /// Get address code. fn code(&self, address: &Address) -> Option; @@ -416,6 +419,11 @@ impl BlockChainClient for Client { Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) } + fn block_hash(&self, id: BlockId) -> Option { + let chain = self.chain.read().unwrap(); + Self::block_hash(&chain, id) + } + fn code(&self, address: &Address) -> Option { self.state().code(address) } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 153a51216..e2cc1a5c3 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -267,11 +267,18 @@ impl EthFilter for EthFilterClient { None => Ok(Value::Array(vec![] as Vec)), Some(info) => match info.filter { PollFilter::Block => { - //unimplemented!() - to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + let current_number = self.client.chain_info().best_block_number; + let hashes = (info.block_number..current_number).into_iter() + .map(BlockId::Number) + .filter_map(|id| self.client.block_hash(id)) + .collect::>(); + + self.polls.lock().unwrap().update_poll(&index.value(), current_number); + + to_value(&hashes) }, PollFilter::PendingTransaction => { - //unimplemented!() + // TODO: fix implementation to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) }, PollFilter::Logs(mut filter) => { @@ -291,4 +298,12 @@ impl EthFilter for EthFilterClient { } }) } + + fn uninstall_filter(&self, params: Params) -> Result { + from_params::<(Index,)>(params) + .and_then(|(index,)| { + self.polls.lock().unwrap().remove_poll(&index.value()); + to_value(&true) + }) + } } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 7f2928ccd..1d442f908 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -105,6 +105,10 @@ impl BlockChainClient for TestBlockChainClient { Some(U256::zero()) } + fn block_hash(&self, id: BlockId) -> Option { + unimplemented!(); + } + fn code(&self, _address: &Address) -> Option { unimplemented!(); } From b60b84fe67f1b77aee61e7c1f41c96f427c1c28b Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 15:56:41 +0100 Subject: [PATCH 03/89] use transient-hashmap from crates.io --- Cargo.lock | 4 ++-- rpc/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 124e3911e..622ac2e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ "serde_codegen 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "syntex 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", - "transient-hashmap 0.1.0 (git+https://github.com/debris/transient-hashmap)", + "transient-hashmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -795,7 +795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "transient-hashmap" version = "0.1.0" -source = "git+https://github.com/debris/transient-hashmap#e21bf844277785504ddc30ee22d2a709103d4992" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index edc749180..3c9851ed3 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -19,7 +19,7 @@ ethsync = { path = "../sync" } clippy = { version = "0.0.44", optional = true } rustc-serialize = "0.3" serde_macros = { version = "0.6.13", optional = true } -transient-hashmap = { git = "https://github.com/debris/transient-hashmap" } +transient-hashmap = "0.1" [build-dependencies] serde_codegen = { version = "0.6.13", optional = true } From 83b8e7df5a6f4bdee8e0ddc208fe44029d4a2517 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 29 Feb 2016 14:57:41 +0100 Subject: [PATCH 04/89] Initial refactor and block closing. --- ethcore/src/block.rs | 53 +++++++++++++++++++------------------------ ethcore/src/client.rs | 39 +++++++++++++++++++++++++++---- ethcore/src/header.rs | 2 +- parity/main.rs | 1 - 4 files changed, 59 insertions(+), 36 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 70c6c2fb2..28005b17e 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -155,9 +155,9 @@ pub struct OpenBlock<'x> { /// Just like OpenBlock, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields, /// and collected the uncles. /// -/// There is no function available to push a transaction. If you want that you'll need to `reopen()` it. -pub struct ClosedBlock<'x> { - open_block: OpenBlock<'x>, +/// There is no function available to push a transaction. +pub struct ClosedBlock { + block: ExecutedBlock, uncle_bytes: Bytes, } @@ -181,7 +181,7 @@ impl<'x> OpenBlock<'x> { r.block.base.header.set_number(parent.number() + 1); r.block.base.header.set_author(author); r.block.base.header.set_extra_data(extra_data); - r.block.base.header.set_timestamp_now(); + r.block.base.header.set_timestamp_now(parent.timestamp()); engine.populate_from_parent(&mut r.block.base.header, parent); engine.on_new_block(&mut r.block); @@ -259,7 +259,7 @@ impl<'x> OpenBlock<'x> { } /// Turn this into a `ClosedBlock`. A BlockChain must be provided in order to figure out the uncles. - pub fn close(self) -> ClosedBlock<'x> { + pub fn close(self) -> ClosedBlock { let mut s = self; s.engine.on_close_block(&mut s.block); s.block.base.header.transactions_root = ordered_trie_root(s.block.base.transactions.iter().map(|ref e| e.rlp_bytes().to_vec()).collect()); @@ -271,7 +271,10 @@ impl<'x> OpenBlock<'x> { s.block.base.header.gas_used = s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used); s.block.base.header.note_dirty(); - ClosedBlock::new(s, uncle_bytes) + ClosedBlock { + block: s.block, + uncle_bytes: uncle_bytes, + } } } @@ -279,38 +282,28 @@ impl<'x> IsBlock for OpenBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.block } } -impl<'x> IsBlock for ClosedBlock<'x> { - fn block(&self) -> &ExecutedBlock { &self.open_block.block } +impl<'x> IsBlock for ClosedBlock { + fn block(&self) -> &ExecutedBlock { &self.block } } -impl<'x> ClosedBlock<'x> { - fn new(open_block: OpenBlock<'x>, uncle_bytes: Bytes) -> Self { - ClosedBlock { - open_block: open_block, - uncle_bytes: uncle_bytes, - } - } - +impl ClosedBlock { /// Get the hash of the header without seal arguments. pub fn hash(&self) -> H256 { self.header().rlp_sha3(Seal::Without) } /// Provide a valid seal in order to turn this into a `SealedBlock`. /// /// NOTE: This does not check the validity of `seal` with the engine. - pub fn seal(self, seal: Vec) -> Result { + pub fn seal(self, engine: &Engine, seal: Vec) -> Result { let mut s = self; - if seal.len() != s.open_block.engine.seal_fields() { - return Err(BlockError::InvalidSealArity(Mismatch{expected: s.open_block.engine.seal_fields(), found: seal.len()})); + if seal.len() != engine.seal_fields() { + return Err(BlockError::InvalidSealArity(Mismatch{expected: engine.seal_fields(), found: seal.len()})); } - s.open_block.block.base.header.set_seal(seal); - Ok(SealedBlock { block: s.open_block.block, uncle_bytes: s.uncle_bytes }) + s.block.base.header.set_seal(seal); + Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }) } - /// Turn this back into an `OpenBlock`. - pub fn reopen(self) -> OpenBlock<'x> { self.open_block } - /// Drop this object and return the underlieing database. - pub fn drain(self) -> JournalDB { self.open_block.block.state.drop().1 } + pub fn drain(self) -> JournalDB { self.block.state.drop().1 } } impl SealedBlock { @@ -332,7 +325,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact<'x>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { +pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); @@ -350,14 +343,14 @@ pub fn enact<'x>(header: &Header, transactions: &[SignedTransaction], uncles: &[ } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes<'x>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { +pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified<'x>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { +pub fn enact_verified(block: &PreVerifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } @@ -365,7 +358,7 @@ pub fn enact_verified<'x>(block: &PreVerifiedBlock, engine: &'x Engine, db: Jour /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); - Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) + Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal()))) } #[cfg(test)] @@ -386,7 +379,7 @@ mod tests { let last_hashes = vec![genesis_header.hash()]; let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); - let _ = b.seal(vec![]); + let _ = b.seal(engine.deref(), vec![]); } #[test] diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 5d71eed84..c378fa449 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -193,6 +193,9 @@ pub struct Client { report: RwLock, import_lock: Mutex<()>, panic_handler: Arc, + + // for sealing... + _sealing_block: Mutex>, } const HISTORY: u64 = 1000; @@ -228,7 +231,8 @@ impl Client { block_queue: RwLock::new(block_queue), report: RwLock::new(Default::default()), import_lock: Mutex::new(()), - panic_handler: panic_handler + panic_handler: panic_handler, + _sealing_block: Mutex::new(None), })) } @@ -237,10 +241,10 @@ impl Client { self.block_queue.write().unwrap().flush(); } - fn build_last_hashes(&self, header: &Header) -> LastHashes { + fn build_last_hashes(&self, parent_hash: H256) -> LastHashes { let mut last_hashes = LastHashes::new(); last_hashes.resize(256, H256::new()); - last_hashes[0] = header.parent_hash.clone(); + last_hashes[0] = parent_hash; let chain = self.chain.read().unwrap(); for i in 0..255 { match chain.block_details(&last_hashes[i]) { @@ -273,7 +277,7 @@ impl Client { // Enact Verified Block let parent = chain_has_parent.unwrap(); - let last_hashes = self.build_last_hashes(header); + let last_hashes = self.build_last_hashes(header.parent_hash.clone()); let db = self.state_db.lock().unwrap().clone(); let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); @@ -302,6 +306,8 @@ impl Client { let _import_lock = self.import_lock.lock(); let blocks = self.block_queue.write().unwrap().drain(max_blocks_to_import); + let original_best = self.chain_info().best_block_hash; + for block in blocks { let header = &block.header; @@ -357,6 +363,10 @@ impl Client { } } + if self.chain_info().best_block_hash != original_best { + self.new_chain_head(); + } + imported } @@ -403,8 +413,29 @@ impl Client { BlockId::Latest => Some(self.chain.read().unwrap().best_block_number()) } } + + /// New chain head event. + pub fn new_chain_head(&self) { + let h = self.chain.read().unwrap().best_block_hash(); + info!("NEW CHAIN HEAD: #{}: {}", self.chain.read().unwrap().best_block_number(), h); + + info!("Preparing to seal."); + let b = OpenBlock::new( + &self.engine, + self.state_db.lock(), + self.chain.read().unwrap().block_header(&h).unwrap_or_else(|| {return;}), + self.build_last_hashes(h.clone()), + x!("0037a6b811ffeb6e072da21179d11b1406371c63"), + b"Parity".to_owned() + ); + let b = b.close(); + info!("Sealed: hash={}, diff={}, number={}", b.hash(), b.block().difficulty(), b.block().number()); + *self._sealing_block.lock().unwrap() = Some(b); + } } +// TODO: need MinerService MinerIoHandler + impl BlockChainClient for Client { fn block_header(&self, id: BlockId) -> Option { let chain = self.chain.read().unwrap(); diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index 6ee682544..3db0be5ff 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -131,7 +131,7 @@ impl Header { /// Set the timestamp field of the header. pub fn set_timestamp(&mut self, a: u64) { self.timestamp = a; self.note_dirty(); } /// Set the timestamp field of the header to the current time. - pub fn set_timestamp_now(&mut self) { self.timestamp = now_utc().to_timespec().sec as u64; self.note_dirty(); } + pub fn set_timestamp_now(&mut self, but_later_than: u64) { self.timestamp = max(now_utc().to_timespec().sec as u64, but_later_than + 1); self.note_dirty(); } /// Set the author field of the header. pub fn set_author(&mut self, a: Address) { if a != self.author { self.author = a; self.note_dirty(); } } diff --git a/parity/main.rs b/parity/main.rs index 6415dfc33..ff86b1663 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -354,7 +354,6 @@ impl Default for Informant { } impl Informant { - fn format_bytes(b: usize) -> String { match binary_prefix(b as f64) { Standalone(bytes) => format!("{} bytes", bytes), From e20858a5dc048c06bac6d3e744979e9fc3ff1639 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 29 Feb 2016 15:30:08 +0100 Subject: [PATCH 05/89] Compile fixes. --- ethcore/src/client.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c378fa449..a4ef970a5 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -21,7 +21,7 @@ use util::panics::*; use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; -use header::{BlockNumber, Header}; +use header::{BlockNumber}; use state::State; use spec::Spec; use engine::Engine; @@ -421,15 +421,15 @@ impl Client { info!("Preparing to seal."); let b = OpenBlock::new( - &self.engine, - self.state_db.lock(), - self.chain.read().unwrap().block_header(&h).unwrap_or_else(|| {return;}), + self.engine.deref().deref(), + self.state_db.lock().unwrap().clone(), + match self.chain.read().unwrap().block_header(&h) { Some(ref x) => x, None => {return;} }, self.build_last_hashes(h.clone()), x!("0037a6b811ffeb6e072da21179d11b1406371c63"), - b"Parity".to_owned() + b"Parity".to_vec() ); let b = b.close(); - info!("Sealed: hash={}, diff={}, number={}", b.hash(), b.block().difficulty(), b.block().number()); + info!("Sealed: hash={}, diff={}, number={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); *self._sealing_block.lock().unwrap() = Some(b); } } From ffc5c2ea7bf5747acbbc87dca25e37943312d771 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 29 Feb 2016 19:30:13 +0100 Subject: [PATCH 06/89] eth_getwork implemented. --- Cargo.lock | 1 + ethash/src/compute.rs | 3 ++- ethash/src/lib.rs | 4 ++-- ethcore/src/client.rs | 13 +++++++------ ethcore/src/ethereum/ethash.rs | 14 ++++++++++---- rpc/Cargo.toml | 1 + rpc/src/v1/impls/eth.rs | 27 +++++++++++++++++++++++++++ util/src/misc.rs | 2 +- util/src/uint.rs | 27 +++++++++++++++++++++++++++ 9 files changed, 78 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bca236813..9c225dbbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,7 @@ name = "ethcore-rpc" version = "0.9.99" dependencies = [ "clippy 0.0.44 (registry+https://github.com/rust-lang/crates.io-index)", + "ethash 0.9.99", "ethcore 0.9.99", "ethcore-util 0.9.99", "ethsync 0.9.99", diff --git a/ethash/src/compute.rs b/ethash/src/compute.rs index 6c311993b..eb64151d0 100644 --- a/ethash/src/compute.rs +++ b/ethash/src/compute.rs @@ -172,7 +172,8 @@ fn get_data_size(block_number: u64) -> usize { } #[inline] -fn get_seedhash(block_number: u64) -> H256 { +/// Given the `block_number`, determine the seed hash for Ethash. +pub fn get_seedhash(block_number: u64) -> H256 { let epochs = block_number / ETHASH_EPOCH_LENGTH; let mut ret: H256 = [0u8; 32]; for _ in 0..epochs { diff --git a/ethash/src/lib.rs b/ethash/src/lib.rs index e96b98b81..982f7129b 100644 --- a/ethash/src/lib.rs +++ b/ethash/src/lib.rs @@ -24,7 +24,7 @@ mod compute; use std::mem; use compute::Light; -pub use compute::{quick_get_difficulty, H256, ProofOfWork, ETHASH_EPOCH_LENGTH}; +pub use compute::{get_seedhash, quick_get_difficulty, H256, ProofOfWork, ETHASH_EPOCH_LENGTH}; use std::sync::{Arc, Mutex}; @@ -35,7 +35,7 @@ struct LightCache { prev: Option>, } -/// Lighy/Full cache manager +/// Light/Full cache manager. pub struct EthashManager { cache: Mutex, } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index d81507d25..f634579ad 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -195,7 +195,7 @@ pub struct Client { panic_handler: Arc, // for sealing... - _sealing_block: Mutex>, + sealing_block: Mutex>, } const HISTORY: u64 = 1000; @@ -232,7 +232,7 @@ impl Client { report: RwLock::new(Default::default()), import_lock: Mutex::new(()), panic_handler: panic_handler, - _sealing_block: Mutex::new(None), + sealing_block: Mutex::new(None), })) } @@ -417,9 +417,7 @@ impl Client { /// New chain head event. pub fn new_chain_head(&self) { let h = self.chain.read().unwrap().best_block_hash(); - info!("NEW CHAIN HEAD: #{}: {}", self.chain.read().unwrap().best_block_number(), h); - - info!("Preparing to seal."); + info!("New best block: #{}: {}", self.chain.read().unwrap().best_block_number(), h); let b = OpenBlock::new( self.engine.deref().deref(), self.state_db.lock().unwrap().clone(), @@ -430,8 +428,11 @@ impl Client { ); let b = b.close(); info!("Sealed: hash={}, diff={}, number={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); - *self._sealing_block.lock().unwrap() = Some(b); + *self.sealing_block.lock().unwrap() = Some(b); } + + /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. + pub fn sealing_block(&self) -> &Mutex> { &self.sealing_block } } // TODO: need MinerService MinerIoHandler diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 54f02524d..4f1ef1ce3 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -144,9 +144,9 @@ impl Engine for Ethash { } let difficulty = Ethash::boundary_to_difficulty(&Ethash::from_ethash(quick_get_difficulty( - &Ethash::to_ethash(header.bare_hash()), - header.nonce().low_u64(), - &Ethash::to_ethash(header.mix_hash())))); + &Ethash::to_ethash(header.bare_hash()), + header.nonce().low_u64(), + &Ethash::to_ethash(header.mix_hash()) ))); if difficulty < header.difficulty { return Err(From::from(BlockError::InvalidProofOfWork(OutOfBounds { min: Some(header.difficulty), max: None, found: difficulty }))); } @@ -241,10 +241,16 @@ impl Ethash { target } - fn boundary_to_difficulty(boundary: &H256) -> U256 { + /// Convert an Ethash boundary to its original difficulty. Basically just `f(x) = 2^256 / x`. + pub fn boundary_to_difficulty(boundary: &H256) -> U256 { U256::from((U512::one() << 256) / x!(U256::from(boundary.as_slice()))) } + /// Convert an Ethash difficulty to the target boundary. Basically just `f(x) = 2^256 / x`. + pub fn difficulty_to_boundary(difficulty: &U256) -> H256 { + x!(U256::from((U512::one() << 256) / x!(difficulty))) + } + fn to_ethash(hash: H256) -> EH256 { unsafe { mem::transmute(hash) } } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index b9ca9deac..c38684643 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -15,6 +15,7 @@ jsonrpc-core = "1.2" jsonrpc-http-server = "2.1" ethcore-util = { path = "../util" } ethcore = { path = "../ethcore" } +ethash = { path = "../ethash" } ethsync = { path = "../sync" } clippy = { version = "0.0.44", optional = true } rustc-serialize = "0.3" diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 00bce5437..5791b7418 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -22,7 +22,11 @@ use util::hash::*; use util::uint::*; use util::sha3::*; use ethcore::client::*; +use ethcore::block::{IsBlock}; use ethcore::views::*; +extern crate ethash; +use self::ethash::get_seedhash; +use ethcore::ethereum::Ethash; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, OptionalValue, Index, Filter, Log}; @@ -209,6 +213,29 @@ impl Eth for EthClient { to_value(&logs) }) } + + fn work(&self, params: Params) -> Result { + match params { + Params::None => { + let c = take_weak!(self.client); + let u = c.sealing_block().lock().unwrap(); + match *u { + Some(ref b) => { + let current_hash = b.hash(); + let target = Ethash::difficulty_to_boundary(b.block().header().difficulty()); + let seed_hash = get_seedhash(b.block().header().number()); + to_value(&(current_hash, seed_hash, target)) + } + _ => Err(Error::invalid_params()) + } + }, + _ => Err(Error::invalid_params()) + } + } + +// fn submit_work(&self, _: Params) -> Result { rpc_unimplemented!() } + +// fn submit_hashrate(&self, _: Params) -> Result { rpc_unimplemented!() } } /// Eth filter rpc implementation. diff --git a/util/src/misc.rs b/util/src/misc.rs index 289f1c50c..35e1f3a75 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -69,5 +69,5 @@ pub fn contents(name: &str) -> Result { /// Get the standard version string for this software. pub fn version() -> String { - format!("Parity//{}-{}-{}/{}-{}-{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date().replace("-", ""), Target::arch(), Target::os(), Target::env(), rustc_version::version()) + format!("Parity//v{}-{}-{}/{}-{}-{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date().replace("-", ""), Target::arch(), Target::os(), Target::env(), rustc_version::version()) } \ No newline at end of file diff --git a/util/src/uint.rs b/util/src/uint.rs index 517b7a29f..7f2e3d3b1 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -1124,6 +1124,33 @@ impl From for U256 { } } +impl<'a> From<&'a U256> for U512 { + fn from(value: &'a U256) -> U512 { + let U256(ref arr) = *value; + let mut ret = [0; 8]; + ret[0] = arr[0]; + ret[1] = arr[1]; + ret[2] = arr[2]; + ret[3] = arr[3]; + U512(ret) + } +} + +impl<'a> From<&'a U512> for U256 { + fn from(value: &'a U512) -> U256 { + let U512(ref arr) = *value; + if arr[4] | arr[5] | arr[6] | arr[7] != 0 { + panic!("Overflow"); + } + let mut ret = [0; 4]; + ret[0] = arr[0]; + ret[1] = arr[1]; + ret[2] = arr[2]; + ret[3] = arr[3]; + U256(ret) + } +} + impl From for U128 { fn from(value: U256) -> U128 { let U256(ref arr) = value; From 394e9c679bfb0e49c51a67b05001043c648a76cc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 00:02:48 +0100 Subject: [PATCH 07/89] Reorganised ImportError to be a type of Errpr (rather than vice-versa). Added support for eth_submitWork. --- ethcore/src/block.rs | 16 ++++++++++++-- ethcore/src/block_queue.rs | 24 ++++++++++---------- ethcore/src/client.rs | 40 +++++++++++++++++++++++++++------- ethcore/src/engine.rs | 2 -- ethcore/src/error.rs | 34 ++++++++++++++++------------- ethcore/src/ethereum/ethash.rs | 9 ++++---- ethcore/src/verification.rs | 8 +++---- rpc/src/v1/impls/eth.rs | 13 ++++++++--- sync/src/chain.rs | 12 +++++----- 9 files changed, 102 insertions(+), 56 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 28005b17e..f72c2f800 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -21,7 +21,7 @@ use common::*; use engine::*; use state::*; -use verification::PreVerifiedBlock; +use verification::PreverifiedBlock; /// A block, encoded as it is on the block chain. // TODO: rename to Block @@ -302,6 +302,18 @@ impl ClosedBlock { Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }) } + /// Provide a valid seal in order to turn this into a `SealedBlock`. + /// This does check the validity of `seal` with the engine. + /// Returns the `ClosedBlock` back again if the seal is no good. + pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { + let mut s = self; + s.block.base.header.set_seal(seal); + match engine.verify_block_basic(&s.block.base.header, None).is_err() || engine.verify_block_unordered(&s.block.base.header, None).is_err() { + false => Err(s), + true => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), + } + } + /// Drop this object and return the underlieing database. pub fn drain(self) -> JournalDB { self.block.state.drop().1 } } @@ -350,7 +362,7 @@ pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: & } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified(block: &PreVerifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 668c004e5..143f53647 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -28,7 +28,7 @@ use service::*; use client::BlockStatus; use util::panics::*; -known_heap_size!(0, UnVerifiedBlock, VerifyingBlock, PreVerifiedBlock); +known_heap_size!(0, UnverifiedBlock, VerifyingBlock, PreverifiedBlock); const MIN_MEM_LIMIT: usize = 16384; const MIN_QUEUE_LIMIT: usize = 512; @@ -105,14 +105,14 @@ pub struct BlockQueue { max_mem_use: usize, } -struct UnVerifiedBlock { +struct UnverifiedBlock { header: Header, bytes: Bytes, } struct VerifyingBlock { hash: H256, - block: Option, + block: Option, } struct QueueSignal { @@ -134,8 +134,8 @@ impl QueueSignal { #[derive(Default)] struct Verification { - unverified: VecDeque, - verified: VecDeque, + unverified: VecDeque, + verified: VecDeque, verifying: VecDeque, bad: HashSet, } @@ -244,7 +244,7 @@ impl BlockQueue { } } - fn drain_verifying(verifying: &mut VecDeque, verified: &mut VecDeque, bad: &mut HashSet) { + fn drain_verifying(verifying: &mut VecDeque, verified: &mut VecDeque, bad: &mut HashSet) { while !verifying.is_empty() && verifying.front().unwrap().block.is_some() { let block = verifying.pop_front().unwrap().block.unwrap(); if bad.contains(&block.header.parent_hash) { @@ -289,31 +289,31 @@ impl BlockQueue { let header = BlockView::new(&bytes).header(); let h = header.hash(); if self.processing.read().unwrap().contains(&h) { - return Err(ImportError::AlreadyQueued); + return Err(x!(ImportError::AlreadyQueued)); } { let mut verification = self.verification.lock().unwrap(); if verification.bad.contains(&h) { - return Err(ImportError::Bad(None)); + return Err(x!(ImportError::KnownBad)); } if verification.bad.contains(&header.parent_hash) { verification.bad.insert(h.clone()); - return Err(ImportError::Bad(None)); + return Err(x!(ImportError::KnownBad)); } } match verify_block_basic(&header, &bytes, self.engine.deref().deref()) { Ok(()) => { self.processing.write().unwrap().insert(h.clone()); - self.verification.lock().unwrap().unverified.push_back(UnVerifiedBlock { header: header, bytes: bytes }); + self.verification.lock().unwrap().unverified.push_back(UnverifiedBlock { header: header, bytes: bytes }); self.more_to_verify.notify_all(); Ok(h) }, Err(err) => { warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), err); self.verification.lock().unwrap().bad.insert(h.clone()); - Err(From::from(err)) + Err(err) } } } @@ -352,7 +352,7 @@ impl BlockQueue { } /// Removes up to `max` verified blocks from the queue - pub fn drain(&mut self, max: usize) -> Vec { + pub fn drain(&mut self, max: usize) -> Vec { let mut verification = self.verification.lock().unwrap(); let count = min(max, verification.verified.len()); let mut result = Vec::with_capacity(count); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index f634579ad..d4fdb66c4 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -176,7 +176,7 @@ pub struct ClientReport { impl ClientReport { /// Alter internal reporting to reflect the additional `block` has been processed. - pub fn accrue_block(&mut self, block: &PreVerifiedBlock) { + pub fn accrue_block(&mut self, block: &PreverifiedBlock) { self.blocks_imported += 1; self.transactions_applied += block.transactions.len(); self.gas_processed = self.gas_processed + block.header.gas_used; @@ -257,7 +257,7 @@ impl Client { last_hashes } - fn check_and_close_block(&self, block: &PreVerifiedBlock) -> Result { + fn check_and_close_block(&self, block: &PreverifiedBlock) -> Result { let engine = self.engine.deref().deref(); let header = &block.header; @@ -433,6 +433,28 @@ impl Client { /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. pub fn sealing_block(&self) -> &Mutex> { &self.sealing_block } + + /// Submit `seal` as a valid solution for the header of `pow_hash`. + pub fn submit_seal(&self, pow_hash: H256, seal: Vec) -> Result<(), Error> { + let mut maybe_b = self.sealing_block.lock().unwrap(); + match *maybe_b { + Some(ref b) if b.hash() == pow_hash => {} + _ => { return Err(Error::PowHashInvalid); } + } + + let b = maybe_b.take(); + match b.unwrap().try_seal(self.engine.deref().deref(), seal) { + Err(old) => { + *maybe_b = Some(old); + Err(Error::PowInvalid) + } + Ok(sealed) => { + // TODO: commit DB from `sealed.drain` and make a VerifiedBlock to skip running the transactions twice. + try!(self.import_block(sealed.rlp_bytes())); + Ok(()) + } + } + } } // TODO: need MinerService MinerIoHandler @@ -509,12 +531,14 @@ impl BlockChainClient for Client { } fn import_block(&self, bytes: Bytes) -> ImportResult { - let header = BlockView::new(&bytes).header(); - if self.chain.read().unwrap().is_known(&header.hash()) { - return Err(ImportError::AlreadyInChain); - } - if self.block_status(BlockId::Hash(header.parent_hash)) == BlockStatus::Unknown { - return Err(ImportError::UnknownParent); + { + let header = BlockView::new(&bytes).header_view(); + if self.chain.read().unwrap().is_known(&header.sha3()) { + return Err(x!(ImportError::AlreadyInChain)); + } + if self.block_status(BlockId::Hash(header.parent_hash())) == BlockStatus::Unknown { + return Err(x!(BlockError::UnknownParent(header.parent_hash()))); + } } self.block_queue.write().unwrap().import_block(bytes) } diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index f86c943ee..c57be3cdb 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -30,8 +30,6 @@ pub trait Engine : Sync + Send { /// The number of additional header fields required for this engine. fn seal_fields(&self) -> usize { 0 } - /// Default values of the additional fields RLP-encoded in a raw (non-list) harness. - fn seal_rlp(&self) -> Bytes { vec![] } /// Additional engine-specific information for the user/developer concerning `header`. fn extra_info(&self, _header: &Header) -> HashMap { HashMap::new() } diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index f75f338bf..824d8da90 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -131,25 +131,14 @@ pub enum BlockError { #[derive(Debug)] /// Import to the block queue result pub enum ImportError { - /// Bad block detected - Bad(Option), - /// Already in the block chain + /// Already in the block chain. AlreadyInChain, - /// Already in the block queue + /// Already in the block queue. AlreadyQueued, - /// Unknown parent - UnknownParent, + /// Already marked as bad from a previous import (could mean parent is bad). + KnownBad, } -impl From for ImportError { - fn from(err: Error) -> ImportError { - ImportError::Bad(Some(err)) - } -} - -/// Result of import block operation. -pub type ImportResult = Result; - #[derive(Debug)] /// General error type which should be capable of representing all errors in ethcore. pub enum Error { @@ -163,14 +152,29 @@ pub enum Error { Execution(ExecutionError), /// Error concerning transaction processing. Transaction(TransactionError), + /// Error concerning block import. + Import(ImportError), + /// PoW hash is invalid or out of date. + PowHashInvalid, + /// The value of the nonce or mishash is invalid. + PowInvalid, } +/// Result of import block operation. +pub type ImportResult = Result; + impl From for Error { fn from(err: TransactionError) -> Error { Error::Transaction(err) } } +impl From for Error { + fn from(err: ImportError) -> Error { + Error::Import(err) + } +} + impl From for Error { fn from(err: BlockError) -> Error { Error::Block(err) diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 4f1ef1ce3..d26a7e2af 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -74,8 +74,6 @@ impl Engine for Ethash { fn version(&self) -> SemanticVersion { SemanticVersion::new(1, 0, 0) } // Two fields - mix fn seal_fields(&self) -> usize { 2 } - // Two empty data items in RLP. - fn seal_rlp(&self) -> Bytes { encode(&H64::new()).to_vec() } /// Additional engine-specific information for the user/developer concerning `header`. fn extra_info(&self, _header: &Header) -> HashMap { HashMap::new() } @@ -261,12 +259,15 @@ impl Ethash { } impl Header { - fn nonce(&self) -> H64 { + pub fn nonce(&self) -> H64 { decode(&self.seal()[1]) } - fn mix_hash(&self) -> H256 { + pub fn mix_hash(&self) -> H256 { decode(&self.seal()[0]) } + pub fn set_nonce_and_mix_hash(&mut self, nonce: &H64, mix_hash: &H256) { + self.seal = vec![encode(mix_hash).to_vec(), encode(nonce).to_vec()]; + } } #[cfg(test)] diff --git a/ethcore/src/verification.rs b/ethcore/src/verification.rs index ad5efd24a..f52e2e1e4 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification.rs @@ -26,7 +26,7 @@ use engine::Engine; use blockchain::*; /// Preprocessed block data gathered in `verify_block_unordered` call -pub struct PreVerifiedBlock { +pub struct PreverifiedBlock { /// Populated block header pub header: Header, /// Populated block transactions @@ -55,8 +55,8 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res /// Phase 2 verification. Perform costly checks such as transaction signatures and block nonce for ethash. /// Still operates on a individual block -/// Returns a PreVerifiedBlock structure populated with transactions -pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result { +/// Returns a PreverifiedBlock structure populated with transactions +pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result { try!(engine.verify_block_unordered(&header, Some(&bytes))); for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { try!(engine.verify_block_unordered(&u, None)); @@ -70,7 +70,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> transactions.push(t); } } - Ok(PreVerifiedBlock { + Ok(PreverifiedBlock { header: header, transactions: transactions, bytes: bytes, diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 5791b7418..1b5c4739c 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -21,6 +21,7 @@ use jsonrpc_core::*; use util::hash::*; use util::uint::*; use util::sha3::*; +use util::rlp::encode; use ethcore::client::*; use ethcore::block::{IsBlock}; use ethcore::views::*; @@ -221,10 +222,10 @@ impl Eth for EthClient { let u = c.sealing_block().lock().unwrap(); match *u { Some(ref b) => { - let current_hash = b.hash(); + let pow_hash = b.hash(); let target = Ethash::difficulty_to_boundary(b.block().header().difficulty()); let seed_hash = get_seedhash(b.block().header().number()); - to_value(&(current_hash, seed_hash, target)) + to_value(&(pow_hash, seed_hash, target)) } _ => Err(Error::invalid_params()) } @@ -233,7 +234,13 @@ impl Eth for EthClient { } } -// fn submit_work(&self, _: Params) -> Result { rpc_unimplemented!() } + fn submit_work(&self, params: Params) -> Result { + from_params::<(H64, H256, H256)>(params).and_then(|(nonce, pow_hash, mix_hash)| { + let c = take_weak!(self.client); + let seal = vec![encode(&mix_hash).to_vec(), encode(&nonce).to_vec()]; + to_value(&c.submit_seal(pow_hash, seal).is_ok()) + }) + } // fn submit_hashrate(&self, _: Params) -> Result { rpc_unimplemented!() } } diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 6a7add27f..4b8065bc7 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -477,19 +477,19 @@ impl ChainSync { // TODO: Decompose block and add to self.headers and self.bodies instead if header.number == From::from(self.current_base_block() + 1) { match io.chain().import_block(block_rlp.as_raw().to_vec()) { - Err(ImportError::AlreadyInChain) => { + Err(Error::Import(ImportError::AlreadyInChain)) => { trace!(target: "sync", "New block already in chain {:?}", h); }, - Err(ImportError::AlreadyQueued) => { + Err(Error::Import(ImportError::AlreadyQueued)) => { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { self.last_imported_block = Some(header.number); trace!(target: "sync", "New block queued {:?}", h); }, - Err(ImportError::UnknownParent) => { + Err(Error::Block(BlockError::UnknownParent(p))) => { unknown = true; - trace!(target: "sync", "New block with unknown parent {:?}", h); + trace!(target: "sync", "New block with unknown parent ({:?}) {:?}", p, h); }, Err(e) => { debug!(target: "sync", "Bad new block {:?} : {:?}", h, e); @@ -781,12 +781,12 @@ impl ChainSync { } match io.chain().import_block(block_rlp.out()) { - Err(ImportError::AlreadyInChain) => { + Err(Error::Import(ImportError::AlreadyInChain)) => { trace!(target: "sync", "Block already in chain {:?}", h); self.last_imported_block = Some(headers.0 + i as BlockNumber); self.last_imported_hash = Some(h.clone()); }, - Err(ImportError::AlreadyQueued) => { + Err(Error::Import(ImportError::AlreadyQueued)) => { trace!(target: "sync", "Block already queued {:?}", h); self.last_imported_block = Some(headers.0 + i as BlockNumber); self.last_imported_hash = Some(h.clone()); From 2266d74c2aec90949803e8721e20136ae5e79d08 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 01:15:00 +0100 Subject: [PATCH 08/89] Fix JSONRPC I/O. --- Cargo.lock | 1 + ethcore/src/ethereum/ethash.rs | 10 ++++++++++ rpc/Cargo.toml | 1 + rpc/src/v1/impls/eth.rs | 15 +++++++++++---- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c225dbbe..b8a92fc3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,6 +209,7 @@ dependencies = [ "ethsync 0.9.99", "jsonrpc-core 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde_codegen 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index d26a7e2af..6dabe558f 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -249,6 +249,11 @@ impl Ethash { x!(U256::from((U512::one() << 256) / x!(difficulty))) } + /// Given the `block_number`, determine the seed hash for Ethash. + pub fn get_seedhash(number: BlockNumber) -> H256 { + Self::from_ethash(ethash::get_seedhash(number)) + } + fn to_ethash(hash: H256) -> EH256 { unsafe { mem::transmute(hash) } } @@ -259,12 +264,17 @@ impl Ethash { } impl Header { + /// Get the none field of the header. pub fn nonce(&self) -> H64 { decode(&self.seal()[1]) } + + /// Get the mix hash field of the header. pub fn mix_hash(&self) -> H256 { decode(&self.seal()[0]) } + + /// Set the nonce and mix hash fields of the header. pub fn set_nonce_and_mix_hash(&mut self, nonce: &H64, mix_hash: &H256) { self.seal = vec![encode(mix_hash).to_vec(), encode(nonce).to_vec()]; } diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c38684643..086fb19c1 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -9,6 +9,7 @@ build = "build.rs" [lib] [dependencies] +log = "0.3" serde = "0.7.0" serde_json = "0.7.0" jsonrpc-core = "1.2" diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 1b5c4739c..b26f38440 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -25,8 +25,7 @@ use util::rlp::encode; use ethcore::client::*; use ethcore::block::{IsBlock}; use ethcore::views::*; -extern crate ethash; -use self::ethash::get_seedhash; +//#[macro_use] extern crate log; use ethcore::ethereum::Ethash; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; @@ -216,6 +215,7 @@ impl Eth for EthClient { } fn work(&self, params: Params) -> Result { + println!("Work wanted: {:?}", params); match params { Params::None => { let c = take_weak!(self.client); @@ -224,7 +224,7 @@ impl Eth for EthClient { Some(ref b) => { let pow_hash = b.hash(); let target = Ethash::difficulty_to_boundary(b.block().header().difficulty()); - let seed_hash = get_seedhash(b.block().header().number()); + let seed_hash = Ethash::get_seedhash(b.block().header().number()); to_value(&(pow_hash, seed_hash, target)) } _ => Err(Error::invalid_params()) @@ -235,6 +235,7 @@ impl Eth for EthClient { } fn submit_work(&self, params: Params) -> Result { + println!("Work submission: {:?}", params); from_params::<(H64, H256, H256)>(params).and_then(|(nonce, pow_hash, mix_hash)| { let c = take_weak!(self.client); let seal = vec![encode(&mix_hash).to_vec(), encode(&nonce).to_vec()]; @@ -242,7 +243,13 @@ impl Eth for EthClient { }) } -// fn submit_hashrate(&self, _: Params) -> Result { rpc_unimplemented!() } + fn submit_hashrate(&self, params: Params) -> Result { + println!("Hashrate submission: {:?}", params); + from_params::<(Index, H256)>(params).and_then(|(rate, id)| { + println!("Miner {} reports a hash rate of {} H/s", id, rate.value()); + to_value(&true) + }) + } } /// Eth filter rpc implementation. From 5ccb172e73a1bf0b7b16545c0560184beef8d9d8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 01:52:22 +0100 Subject: [PATCH 09/89] Hashrate now reported correctly. --- ethcore/src/client.rs | 4 ++-- rpc/src/v1/impls/eth.rs | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index d4fdb66c4..c5aa987f6 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -417,7 +417,7 @@ impl Client { /// New chain head event. pub fn new_chain_head(&self) { let h = self.chain.read().unwrap().best_block_hash(); - info!("New best block: #{}: {}", self.chain.read().unwrap().best_block_number(), h); + debug!("New best block: #{}: {}", self.chain.read().unwrap().best_block_number(), h); let b = OpenBlock::new( self.engine.deref().deref(), self.state_db.lock().unwrap().clone(), @@ -427,7 +427,7 @@ impl Client { b"Parity".to_vec() ); let b = b.close(); - info!("Sealed: hash={}, diff={}, number={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); + debug!("Sealing: hash={}, diff={}, number={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); *self.sealing_block.lock().unwrap() = Some(b); } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index b26f38440..f96d69ce7 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -15,12 +15,12 @@ // along with Parity. If not, see . //! Eth rpc implementation. -use std::sync::{Arc, Weak}; use ethsync::{EthSync, SyncState}; use jsonrpc_core::*; use util::hash::*; use util::uint::*; use util::sha3::*; +use util::standard::{RwLock, HashMap, Arc, Weak}; use util::rlp::encode; use ethcore::client::*; use ethcore::block::{IsBlock}; @@ -34,7 +34,8 @@ use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncIn /// Eth rpc implementation. pub struct EthClient { client: Weak, - sync: Weak + sync: Weak, + hashrates: RwLock>, } impl EthClient { @@ -42,7 +43,8 @@ impl EthClient { pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { client: Arc::downgrade(client), - sync: Arc::downgrade(sync) + sync: Arc::downgrade(sync), + hashrates: RwLock::new(HashMap::new()), } } @@ -137,7 +139,7 @@ impl Eth for EthClient { // TODO: return real hashrate once we have mining fn hashrate(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::zero()), + Params::None => to_value(&self.hashrates.read().unwrap().iter().fold(0u64, |sum, (_, v)| sum + v)), _ => Err(Error::invalid_params()) } } @@ -215,7 +217,6 @@ impl Eth for EthClient { } fn work(&self, params: Params) -> Result { - println!("Work wanted: {:?}", params); match params { Params::None => { let c = take_weak!(self.client); @@ -235,6 +236,7 @@ impl Eth for EthClient { } fn submit_work(&self, params: Params) -> Result { + // TODO: println! should be debug! println!("Work submission: {:?}", params); from_params::<(H64, H256, H256)>(params).and_then(|(nonce, pow_hash, mix_hash)| { let c = take_weak!(self.client); @@ -244,9 +246,9 @@ impl Eth for EthClient { } fn submit_hashrate(&self, params: Params) -> Result { - println!("Hashrate submission: {:?}", params); + // TODO: Index should be U256. from_params::<(Index, H256)>(params).and_then(|(rate, id)| { - println!("Miner {} reports a hash rate of {} H/s", id, rate.value()); + self.hashrates.write().unwrap().insert(id, rate.value() as u64); to_value(&true) }) } From 1eca9acffb2f952678d7135f059139b650eec289 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 02:33:41 +0100 Subject: [PATCH 10/89] Fix is_mining. --- rpc/src/v1/impls/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index f96d69ce7..24ecbe5f1 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -131,7 +131,7 @@ impl Eth for EthClient { // TODO: return real value of mining once it's implemented. fn is_mining(&self, params: Params) -> Result { match params { - Params::None => Ok(Value::Bool(false)), + Params::None => to_value(&!self.hashrates.read().unwrap().is_empty()), _ => Err(Error::invalid_params()) } } From 48df869202571360b4554544bf4400f19a0236d3 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 16:58:14 +0100 Subject: [PATCH 11/89] --author and --extra-data options. Fixed null parent-hash. --- ethcore/src/block.rs | 12 +++++++--- ethcore/src/client.rs | 50 +++++++++++++++++++++++++++++++++-------- ethcore/src/header.rs | 10 ++++++--- parity/main.rs | 20 +++++++++++++++++ rpc/src/v1/impls/eth.rs | 6 ++--- util/src/misc.rs | 15 +++++++++++++ 6 files changed, 95 insertions(+), 18 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index f72c2f800..16d3e2216 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -182,6 +182,7 @@ impl<'x> OpenBlock<'x> { r.block.base.header.set_author(author); r.block.base.header.set_extra_data(extra_data); r.block.base.header.set_timestamp_now(parent.timestamp()); + r.block.base.header.set_parent_hash(parent.hash()); engine.populate_from_parent(&mut r.block.base.header, parent); engine.on_new_block(&mut r.block); @@ -308,10 +309,15 @@ impl ClosedBlock { pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { let mut s = self; s.block.base.header.set_seal(seal); - match engine.verify_block_basic(&s.block.base.header, None).is_err() || engine.verify_block_unordered(&s.block.base.header, None).is_err() { - false => Err(s), - true => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), + if let Err(e) = engine.verify_block_basic(&s.block.base.header, None) { + debug!("Failed to try_seal: {:?}", e); + return Err(s); } + if let Err(e) = engine.verify_block_unordered(&s.block.base.header, None) { + debug!("Failed to try_seal: {:?}", e); + return Err(s); + } + Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }) } /// Drop this object and return the underlieing database. diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c5aa987f6..de875348a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -196,6 +196,8 @@ pub struct Client { // for sealing... sealing_block: Mutex>, + author: RwLock
, + extra_data: RwLock, } const HISTORY: u64 = 1000; @@ -233,6 +235,8 @@ impl Client { import_lock: Mutex::new(()), panic_handler: panic_handler, sealing_block: Mutex::new(None), + author: RwLock::new(Address::new()), + extra_data: RwLock::new(Vec::new()), })) } @@ -364,7 +368,7 @@ impl Client { } if self.chain_info().best_block_hash != original_best { - self.new_chain_head(); + self.prepare_sealing(); } imported @@ -414,27 +418,55 @@ impl Client { } } - /// New chain head event. - pub fn new_chain_head(&self) { + /// Set the author that we will seal blocks as. + pub fn author(&self) -> Address { + self.author.read().unwrap().clone() + } + + /// Set the author that we will seal blocks as. + pub fn set_author(&self, author: Address) { + *self.author.write().unwrap() = author; + } + + /// Set the extra_data that we will seal blocks wuth. + pub fn extra_data(&self) -> Bytes { + self.extra_data.read().unwrap().clone() + } + + /// Set the extra_data that we will seal blocks with. + pub fn set_extra_data(&self, extra_data: Bytes) { + *self.extra_data.write().unwrap() = extra_data; + } + + /// New chain head event. Restart mining operation. + pub fn prepare_sealing(&self) { let h = self.chain.read().unwrap().best_block_hash(); - debug!("New best block: #{}: {}", self.chain.read().unwrap().best_block_number(), h); let b = OpenBlock::new( self.engine.deref().deref(), self.state_db.lock().unwrap().clone(), match self.chain.read().unwrap().block_header(&h) { Some(ref x) => x, None => {return;} }, - self.build_last_hashes(h.clone()), - x!("0037a6b811ffeb6e072da21179d11b1406371c63"), - b"Parity".to_vec() + self.build_last_hashes(h), + self.author(), + self.extra_data() ); + // TODO: push uncles. + // TODO: push transactions. let b = b.close(); - debug!("Sealing: hash={}, diff={}, number={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); + trace!("Sealing: number={}, hash={}, diff={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); + debug!("Header: {:?}", b.block().header()); *self.sealing_block.lock().unwrap() = Some(b); } /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. - pub fn sealing_block(&self) -> &Mutex> { &self.sealing_block } + pub fn sealing_block(&self) -> &Mutex> { + if self.sealing_block.lock().unwrap().is_none() { + self.prepare_sealing(); + } + &self.sealing_block + } /// Submit `seal` as a valid solution for the header of `pow_hash`. + /// Will check the seal, but not actually insert the block into the chain. pub fn submit_seal(&self, pow_hash: H256, seal: Vec) -> Result<(), Error> { let mut maybe_b = self.sealing_block.lock().unwrap(); match *maybe_b { diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index 3db0be5ff..cc02d84db 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -102,10 +102,12 @@ impl Header { Self::default() } - /// Get the number field of the header. - pub fn number(&self) -> BlockNumber { self.number } + /// Get the parent_hash field of the header. + pub fn parent_hash(&self) -> &H256 { &self.parent_hash } /// Get the timestamp field of the header. pub fn timestamp(&self) -> u64 { self.timestamp } + /// Get the number field of the header. + pub fn number(&self) -> BlockNumber { self.number } /// Get the author field of the header. pub fn author(&self) -> &Address { &self.author } @@ -127,11 +129,13 @@ impl Header { // TODO: seal_at, set_seal_at &c. /// Set the number field of the header. - pub fn set_number(&mut self, a: BlockNumber) { self.number = a; self.note_dirty(); } + pub fn set_parent_hash(&mut self, a: H256) { self.parent_hash = a; self.note_dirty(); } /// Set the timestamp field of the header. pub fn set_timestamp(&mut self, a: u64) { self.timestamp = a; self.note_dirty(); } /// Set the timestamp field of the header to the current time. pub fn set_timestamp_now(&mut self, but_later_than: u64) { self.timestamp = max(now_utc().to_timespec().sec as u64, but_later_than + 1); self.note_dirty(); } + /// Set the number field of the header. + pub fn set_number(&mut self, a: BlockNumber) { self.number = a; self.note_dirty(); } /// Set the author field of the header. pub fn set_author(&mut self, a: Address) { if a != self.author { self.author = a; self.note_dirty(); } } diff --git a/parity/main.rs b/parity/main.rs index 7241a2ac4..b991f36cd 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -85,6 +85,10 @@ Options: --jsonrpc-url URL Specify URL for JSON-RPC API server [default: 127.0.0.1:8545]. --jsonrpc-cors URL Specify CORS header for JSON-RPC API responses [default: null]. + --author ADDRESS Specify the block author (aka "coinbase") address for sending block rewards + from sealed blocks [default: 0037a6b811ffeb6e072da21179d11b1406371c63]. + --extra-data STRING Specify a custom extra-data for authored blocks, no more than 32 characters. + -l --logging LOGGING Specify the logging level. -v --version Show information about version. -h --help Show this screen. @@ -114,6 +118,8 @@ struct Args { flag_jsonrpc_cors: String, flag_logging: Option, flag_version: bool, + flag_author: String, + flag_extra_data: Option, } fn setup_log(init: &Option) { @@ -196,6 +202,18 @@ impl Configuration { self.args.flag_db_path.replace("$HOME", env::home_dir().unwrap().to_str().unwrap()) } + fn author(&self) -> Address { + Address::from_str(&self.args.flag_author).unwrap_or_else(|_| die!("{}: Invalid address for --author. Must be 40 hex characters, without the 0x at the beginning.", self.args.flag_author)) + } + + fn extra_data(&self) -> Bytes { + match self.args.flag_extra_data { + Some(ref x) if x.len() <= 32 => x.as_bytes().to_owned(), + None => version_data(), + Some(ref x) => { die!("{}: Extra data must be at most 32 characters.", x); } + } + } + fn _keys_path(&self) -> String { self.args.flag_keys_path.replace("$HOME", env::home_dir().unwrap().to_str().unwrap()) } @@ -296,6 +314,8 @@ impl Configuration { client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); + client.set_author(self.author()); + client.set_extra_data(self.extra_data()); // Sync let sync = EthSync::register(service.network(), sync_config, client); diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 24ecbe5f1..a7583ddf7 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -236,12 +236,12 @@ impl Eth for EthClient { } fn submit_work(&self, params: Params) -> Result { - // TODO: println! should be debug! - println!("Work submission: {:?}", params); from_params::<(H64, H256, H256)>(params).and_then(|(nonce, pow_hash, mix_hash)| { +// trace!("Decoded: nonce={}, pow_hash={}, mix_hash={}", nonce, pow_hash, mix_hash); let c = take_weak!(self.client); let seal = vec![encode(&mix_hash).to_vec(), encode(&nonce).to_vec()]; - to_value(&c.submit_seal(pow_hash, seal).is_ok()) + let r = c.submit_seal(pow_hash, seal); + to_value(&r.is_ok()) }) } diff --git a/util/src/misc.rs b/util/src/misc.rs index 35e1f3a75..d9f52fd71 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -18,6 +18,7 @@ use std::fs::File; use common::*; +use rlp::{Stream, RlpStream}; use target_info::Target; use rustc_version; @@ -70,4 +71,18 @@ pub fn contents(name: &str) -> Result { /// Get the standard version string for this software. pub fn version() -> String { format!("Parity//v{}-{}-{}/{}-{}-{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date().replace("-", ""), Target::arch(), Target::os(), Target::env(), rustc_version::version()) +} + +/// Get the standard version data for this software. +pub fn version_data() -> Bytes { + let mut s = RlpStream::new_list(4); + let v = + (u32::from_str(env!("CARGO_PKG_VERSION_MAJOR")).unwrap() << 16) + + (u32::from_str(env!("CARGO_PKG_VERSION_MINOR")).unwrap() << 8) + + u32::from_str(env!("CARGO_PKG_VERSION_PATCH")).unwrap(); + s.append(&v); + s.append(&"Parity"); + s.append(&format!("rustc{}", rustc_version::version())); + s.append(&Target::os()); + s.out() } \ No newline at end of file From a5c65b2a3d1aa9a13d680a7efb122be28e47f9b0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 16:59:01 +0100 Subject: [PATCH 12/89] Reduce spam. --- ethcore/src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index de875348a..e5d9f8fcf 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -453,7 +453,6 @@ impl Client { // TODO: push transactions. let b = b.close(); trace!("Sealing: number={}, hash={}, diff={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); - debug!("Header: {:?}", b.block().header()); *self.sealing_block.lock().unwrap() = Some(b); } From afc06050807867ee27a6830aaf15eee29042c60d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 17:23:44 +0100 Subject: [PATCH 13/89] Refactor engine to make it clear that we're actually checking the seal. --- ethcore/src/block.rs | 11 +++-------- ethcore/src/engine.rs | 7 +++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 16d3e2216..eb9d66124 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -309,15 +309,10 @@ impl ClosedBlock { pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { let mut s = self; s.block.base.header.set_seal(seal); - if let Err(e) = engine.verify_block_basic(&s.block.base.header, None) { - debug!("Failed to try_seal: {:?}", e); - return Err(s); + match engine.verify_block_seal(&s.block.base.header) { + Err(_) => Err(s), + _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } - if let Err(e) = engine.verify_block_unordered(&s.block.base.header, None) { - debug!("Failed to try_seal: {:?}", e); - return Err(s); - } - Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }) } /// Drop this object and return the underlieing database. diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index c57be3cdb..a05c78ff5 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -74,6 +74,13 @@ pub trait Engine : Sync + Send { /// Verify a particular transaction is valid. fn verify_transaction(&self, _t: &SignedTransaction, _header: &Header) -> Result<(), Error> { Ok(()) } + /// Verify the seal of a block. This is an auxilliary method that actually just calls other `verify_` methods + /// to get the job done. By default it must pass `verify_basic` and `verify_block_unordered`. If more or fewer + /// methods are needed for an Engine, this may be overridden. + fn verify_block_seal(&self, header: &Header) -> Result<(), Error> { + self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) + } + /// Don't forget to call Super::populateFromParent when subclassing & overriding. // TODO: consider including State in the params. fn populate_from_parent(&self, _header: &mut Header, _parent: &Header) {} From 140711dd8ac6dbd55d3210bc6609711fae73f5a1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 19:59:12 +0100 Subject: [PATCH 14/89] Fixups from review. Reduce size of default extra-data. Introduce find_uncle_headers. --- ethcore/src/block.rs | 2 +- ethcore/src/block_queue.rs | 2 +- ethcore/src/blockchain/blockchain.rs | 6 ++++++ ethcore/src/client.rs | 13 ++++++++----- util/src/misc.rs | 4 ++-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index eb9d66124..582b8669a 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -404,7 +404,7 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(&engine, vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 143f53647..490a17995 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -455,7 +455,7 @@ mod tests { match duplicate_import { Err(e) => { match e { - ImportError::AlreadyQueued => {}, + Error::Import(ImportError::AlreadyQueued) => {}, _ => { panic!("must return AlreadyQueued error"); } } } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 0aba36a8e..88a6504ac 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -473,6 +473,12 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } + /// Given a block's `parent`, find every block header which represents a valid uncle. + pub fn find_uncle_headers(&self, _parent: &H256) -> Vec
{ + // TODO + Vec::new() + } + /// Get inserted block info which is critical to preapre extras updates. fn block_info(&self, block_bytes: &[u8]) -> BlockInfo { let block = BlockView::new(block_bytes); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index e5d9f8fcf..4b109c8a3 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -418,7 +418,7 @@ impl Client { } } - /// Set the author that we will seal blocks as. + /// Get the author that we will seal blocks as. pub fn author(&self) -> Address { self.author.read().unwrap().clone() } @@ -428,7 +428,7 @@ impl Client { *self.author.write().unwrap() = author; } - /// Set the extra_data that we will seal blocks wuth. + /// Get the extra_data that we will seal blocks wuth. pub fn extra_data(&self) -> Bytes { self.extra_data.read().unwrap().clone() } @@ -441,16 +441,19 @@ impl Client { /// New chain head event. Restart mining operation. pub fn prepare_sealing(&self) { let h = self.chain.read().unwrap().best_block_hash(); - let b = OpenBlock::new( + let mut b = OpenBlock::new( self.engine.deref().deref(), self.state_db.lock().unwrap().clone(), match self.chain.read().unwrap().block_header(&h) { Some(ref x) => x, None => {return;} }, - self.build_last_hashes(h), + self.build_last_hashes(h.clone()), self.author(), self.extra_data() ); - // TODO: push uncles. + + self.chain.read().unwrap().find_uncle_headers(&h).into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + // TODO: push transactions. + let b = b.close(); trace!("Sealing: number={}, hash={}, diff={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); *self.sealing_block.lock().unwrap() = Some(b); diff --git a/util/src/misc.rs b/util/src/misc.rs index d9f52fd71..6cd506d02 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -82,7 +82,7 @@ pub fn version_data() -> Bytes { u32::from_str(env!("CARGO_PKG_VERSION_PATCH")).unwrap(); s.append(&v); s.append(&"Parity"); - s.append(&format!("rustc{}", rustc_version::version())); - s.append(&Target::os()); + s.append(&format!("{}", rustc_version::version())); + s.append(&&Target::os()[0..2]); s.out() } \ No newline at end of file From 628a53cceca9ad2b66d8c7510331ad23cc50134c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 20:02:59 +0100 Subject: [PATCH 15/89] Update tests. Fix our tests. --- ethcore/res/ethereum/tests | 2 +- ethcore/src/block.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index f32954b3d..99afe8f5a 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit f32954b3ddb5af2dc3dc9ec6d9a28bee848fdf70 +Subproject commit 99afe8f5aad7bca5d0f1b1685390a4dea32d73c3 diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 582b8669a..0ca6e88be 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -404,7 +404,7 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(&engine, vec![]).unwrap(); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(engine.deref(), vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); From a21fda23a6e1c954f284bfff6cf7ac2797dbec4b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Mar 2016 20:14:28 +0100 Subject: [PATCH 16/89] Enable transition test. --- ethcore/src/json_tests/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index b5f28444a..f6b5751a7 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -115,7 +115,7 @@ declare_test!{StateTests_stSolidityTest, "StateTests/stSolidityTest"} declare_test!{StateTests_stSpecialTest, "StateTests/stSpecialTest"} declare_test!{StateTests_stSystemOperationsTest, "StateTests/stSystemOperationsTest"} declare_test!{StateTests_stTransactionTest, "StateTests/stTransactionTest"} -//declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} +declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} declare_test!{StateTests_stWalletTest, "StateTests/stWalletTest"} From 725e894f9bd26397364943df9d259211bc532844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 1 Mar 2016 21:48:58 +0100 Subject: [PATCH 17/89] TransactionsQueue implementation --- Cargo.lock | 1 + sync/Cargo.toml | 1 + sync/src/lib.rs | 2 + sync/src/transaction_queue.rs | 622 ++++++++++++++++++++++++++++++++++ 4 files changed, 626 insertions(+) create mode 100644 sync/src/transaction_queue.rs diff --git a/Cargo.lock b/Cargo.lock index bca236813..e558606eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -259,6 +259,7 @@ dependencies = [ "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 26a7d463c..2ce65ca77 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -16,6 +16,7 @@ env_logger = "0.3" time = "0.1.34" rand = "0.3.13" heapsize = "0.3" +rustc-serialize = "0.3" [features] default = [] diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 6f28fc320..74541660d 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -70,6 +70,8 @@ use io::NetSyncIo; mod chain; mod io; mod range_collection; +// TODO [todr] Made public to suppress dead code warnings +pub mod transaction_queue; #[cfg(test)] mod tests; diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs new file mode 100644 index 000000000..341607afe --- /dev/null +++ b/sync/src/transaction_queue.rs @@ -0,0 +1,622 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +// TODO [todr] - own transactions should have higher priority + +//! Transaction Queue + +use std::vec::Vec; +use std::cmp::{Ordering}; +use std::collections::{HashMap, BTreeSet}; +use util::uint::{Uint, U256}; +use util::hash::{Address}; +use util::table::*; +use ethcore::transaction::*; + + +#[derive(Clone, Debug)] +struct VerifiedTransaction { + tx: SignedTransaction, + nonce_height: U256 +} + +impl VerifiedTransaction { + pub fn new(tx: SignedTransaction, nonce_height: U256) -> VerifiedTransaction { + VerifiedTransaction { + tx: tx, + nonce_height: nonce_height + } + } + + pub fn sender(&self) -> Address { + self.tx.sender().unwrap() + } +} + +impl Eq for VerifiedTransaction {} +impl PartialEq for VerifiedTransaction { + fn eq(&self, other: &VerifiedTransaction) -> bool { + self.cmp(other) == Ordering::Equal + } +} +impl PartialOrd for VerifiedTransaction { + fn partial_cmp(&self, other: &VerifiedTransaction) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for VerifiedTransaction { + fn cmp(&self, b: &VerifiedTransaction) -> Ordering { + // First check nonce_height + if self.nonce_height != b.nonce_height { + return self.nonce_height.cmp(&b.nonce_height); + } + + // Then compare gas_prices + let a_gas = self.tx.gas_price; + let b_gas = b.tx.gas_price; + if a_gas != b_gas { + return a_gas.cmp(&b_gas); + } + + // Compare nonce + let a_nonce = self.tx.nonce; + let b_nonce = b.tx.nonce; + if a_nonce != b_nonce { + return a_nonce.cmp(&b_nonce); + } + + // and senders + let a_sender = self.sender(); + let b_sender = b.sender(); + a_sender.cmp(&b_sender) + } +} + +struct TransactionsByPriorityAndAddress { + priority: BTreeSet, + address: Table, + limit: usize, +} + +impl TransactionsByPriorityAndAddress { + fn insert(&mut self, address: Address, nonce: U256, verified_tx: VerifiedTransaction) { + self.priority.insert(verified_tx.clone()); + self.address.insert(address, nonce, verified_tx); + } + + fn enforce_limit(&mut self) { + let len = self.priority.len(); + if len <= self.limit { + return; + } + + let to_remove : Vec = { + self.priority + .iter() + .skip(self.limit) + .map(|v_tx| v_tx.tx.clone()) + .collect() + }; + + for tx in to_remove { + self.remove(&tx); + } + } + + fn remove_by_address(&mut self, sender: &Address, nonce: &U256) -> Option { + if let Some(verified_tx) = self.address.remove(sender, nonce) { + self.priority.remove(&verified_tx); + return Some(verified_tx); + } + None + } + + fn remove(&mut self, tx: &SignedTransaction) -> Option { + // First find the transaction by address + let address = tx.sender().unwrap(); + self.remove_by_address(&address, &tx.nonce) + } + + fn clear(&mut self) { + self.priority.clear(); + self.address.clear(); + } +} + +#[derive(Debug)] +/// Current status of the queue +pub struct TransactionQueueStatus { + /// Number of pending transactions (ready to go to block) + pub pending: usize, + /// Number of future transactions (waiting for transactions with lower nonces first) + pub future: usize, +} + +/// TransactionQueue implementation +pub struct TransactionQueue { + /// Priority queue for transactions that can go to block + current: TransactionsByPriorityAndAddress, + /// Priority queue for transactions that has been received but are not yet valid to go to block + future: TransactionsByPriorityAndAddress, + /// Last nonce of transaction in current + last_nonces: HashMap, + /// First nonce of transaction in current (used to determine priority) + first_nonces: HashMap, +} + +impl TransactionQueue { + /// Creates new instance of this Queue + pub fn new() -> Self { + Self::with_limits(1024, 1024) + } + + /// Create new instance of this Queue with specified limits + pub fn with_limits(current_limit: usize, future_limit: usize) -> Self { + let current = TransactionsByPriorityAndAddress { + address: Table::new(), + priority: BTreeSet::new(), + limit: current_limit, + }; + let future = TransactionsByPriorityAndAddress { + address: Table::new(), + priority: BTreeSet::new(), + limit: future_limit, + }; + + TransactionQueue { + current: current, + future: future, + last_nonces: HashMap::new(), + first_nonces: HashMap::new(), + } + } + + /// Returns current status for this queue + pub fn status(&self) -> TransactionQueueStatus { + TransactionQueueStatus { + pending: self.current.priority.len(), + future: self.future.priority.len(), + } + } + + /// Adds all signed transactions to queue to be verified and imported + pub fn add_all(&mut self, txs: Vec, fetch_nonce: T) + where T: Fn(&Address) -> U256 { + for tx in txs.into_iter() { + self.add(tx, &fetch_nonce); + } + } + + /// Add signed transaction to queue to be verified and imported + pub fn add(&mut self, tx: SignedTransaction, fetch_nonce: &T) + where T: Fn(&Address) -> U256 { + self.import_tx(tx, fetch_nonce); + } + + /// Removes all transactions in given slice + /// + /// If gap is introduced marks subsequent transactions as future + pub fn remove_all(&mut self, txs: &[SignedTransaction]) { + for tx in txs { + self.remove(&tx); + } + } + + /// Removes transaction from queue. + /// + /// If gap is introduced marks subsequent transactions as future + pub fn remove(&mut self, tx: &SignedTransaction) { + // Remove from current + let removed = self.current.remove(tx); + if let Some(verified_tx) = removed { + let sender = verified_tx.sender(); + + // Are there any other transactions from this sender? + if !self.current.address.has_row(&sender) { + // Clear last & first nonces + self.last_nonces.remove(&sender); + self.first_nonces.remove(&sender); + return; + } + + // Let's find those with higher nonce (TODO [todr] optimize?) + let to_move_to_future = { + let row_map = self.current.address.row(&sender).unwrap(); + let tx_nonce = verified_tx.tx.nonce; + let mut to_future = Vec::new(); + let mut highest = U256::zero(); + let mut lowest = tx_nonce.clone(); + + // Search nonces to remove and track lowest and highest + for (nonce, _) in row_map.iter() { + if nonce > &tx_nonce { + to_future.push(nonce.clone()); + } else if nonce > &highest { + highest = nonce.clone(); + } else if nonce < &lowest { + lowest = nonce.clone(); + } + } + + // Update first_nonces and last_nonces + if highest == U256::zero() { + self.last_nonces.remove(&sender); + } else { + self.last_nonces.insert(sender.clone(), highest); + } + + if lowest == tx_nonce { + self.first_nonces.remove(&sender); + } else { + self.first_nonces.insert(sender.clone(), lowest); + } + + // return to future + to_future + }; + + for k in to_move_to_future { + if let Some(v) = self.current.remove_by_address(&sender, &k) { + self.future.insert(sender.clone(), v.tx.nonce, v); + } + } + self.future.enforce_limit(); + return; + } + + // Remove from future + { + let sender = tx.sender().unwrap(); + if let Some(_) = self.future.remove_by_address(&sender, &tx.nonce) { + return; + } + } + } + + /// Returns top transactions from the queue + pub fn top_transactions(&self, size: usize) -> Vec { + self.current.priority + .iter() + .take(size) + .map(|t| t.tx.clone()).collect() + } + + /// Removes all elements (in any state) from the queue + pub fn clear(&mut self) { + self.current.clear(); + self.future.clear(); + self.last_nonces.clear(); + self.first_nonces.clear(); + } + + fn move_future_txs(&mut self, address: Address, current_nonce: U256, first_nonce: U256) -> Option { + let mut current_nonce = current_nonce + U256::one(); + { + let txs_by_nonce = self.future.address.row_mut(&address); + if let None = txs_by_nonce { + return None; + } + let mut txs_by_nonce = txs_by_nonce.unwrap(); + + while let Some(tx) = txs_by_nonce.remove(¤t_nonce) { + // remove also from priority + self.future.priority.remove(&tx); + // Put to current + let height = current_nonce - first_nonce; + let verified_tx = VerifiedTransaction::new(tx.tx, U256::from(height)); + self.current.insert(address.clone(), verified_tx.tx.nonce, verified_tx); + current_nonce = current_nonce + U256::one(); + } + } + self.future.address.clear_if_empty(&address); + // Returns last inserted nonce + Some(current_nonce - U256::one()) + } + + fn import_tx(&mut self, tx: SignedTransaction, fetch_nonce: &T) + where T: Fn(&Address) -> U256 { + let nonce = tx.nonce; + let address = tx.sender().unwrap(); + + let next_nonce = U256::one() + self.last_nonces + .get(&address) + .cloned() + .unwrap_or_else(|| fetch_nonce(&address)); + + // Check height + if nonce > next_nonce { + let height = nonce - next_nonce; + let verified_tx = VerifiedTransaction::new(tx, height); + // We have a gap - put to future + self.future.insert(address, nonce, verified_tx); + self.future.enforce_limit(); + return; + } else if next_nonce > nonce { + // Droping transaction + return; + } + + let first_nonce = self.first_nonces + .get(&address) + .cloned() + .unwrap_or_else(|| nonce.clone()); + + let height = nonce - first_nonce; + let verified_tx = VerifiedTransaction::new(tx, height); + // Insert to current + self.current.insert(address.clone(), nonce, verified_tx); + // But maybe there are some more items waiting in future? + let new_last_nonce = self.move_future_txs(address.clone(), nonce, first_nonce); + self.first_nonces.insert(address.clone(), first_nonce); + self.last_nonces.insert(address.clone(), new_last_nonce.unwrap_or(nonce)); + // Enforce limit + self.current.enforce_limit(); + } +} + +#[cfg(test)] +mod test { + extern crate rustc_serialize; + use self::rustc_serialize::hex::FromHex; + + use util::crypto::KeyPair; + use util::uint::{U256, Uint}; + use util::hash::{Address}; + use ethcore::transaction::*; + use super::*; + + fn new_unsigned_tx(nonce: U256) -> Transaction { + Transaction { + action: Action::Create, + value: U256::from(100), + data: "3331600055".from_hex().unwrap(), + gas: U256::from(100_000), + gas_price: U256::one(), + nonce: nonce + } + } + + fn new_tx() -> SignedTransaction { + let keypair = KeyPair::create().unwrap(); + new_unsigned_tx(U256::from(123)).sign(&keypair.secret()) + } + + fn default_nonce(_address: &Address) -> U256 { + U256::from(122) + } + + fn new_txs(second_nonce: U256) -> (SignedTransaction, SignedTransaction) { + let keypair = KeyPair::create().unwrap(); + let secret = &keypair.secret(); + let nonce = U256::from(123); + let tx = new_unsigned_tx(nonce); + let tx2 = new_unsigned_tx(nonce + second_nonce); + + (tx.sign(secret), tx2.sign(secret)) + } + + #[test] + fn should_import_tx() { + // given + let mut txq = TransactionQueue::new(); + let tx = new_tx(); + + // when + txq.add(tx, &default_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 1); + } + + #[test] + fn should_import_txs_from_same_sender() { + // given + let mut txq = TransactionQueue::new(); + + let (tx, tx2) = new_txs(U256::from(1)); + + // when + txq.add(tx.clone(), &default_nonce); + txq.add(tx2.clone(), &default_nonce); + + // then + let top = txq.top_transactions(5); + assert_eq!(top[0], tx); + assert_eq!(top[1], tx2); + assert_eq!(top.len(), 2); + } + + #[test] + fn should_put_transaction_to_futures_if_gap_detected() { + // given + let mut txq = TransactionQueue::new(); + + let (tx, tx2) = new_txs(U256::from(2)); + + // when + txq.add(tx.clone(), &default_nonce); + txq.add(tx2.clone(), &default_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 1); + assert_eq!(stats.future, 1); + let top = txq.top_transactions(5); + assert_eq!(top.len(), 1); + assert_eq!(top[0], tx); + } + + #[test] + fn should_move_transactions_if_gap_filled() { + // given + let mut txq = TransactionQueue::new(); + let kp = KeyPair::create().unwrap(); + let secret = kp.secret(); + let tx = new_unsigned_tx(U256::from(123)).sign(&secret); + let tx1 = new_unsigned_tx(U256::from(124)).sign(&secret); + let tx2 = new_unsigned_tx(U256::from(125)).sign(&secret); + + txq.add(tx, &default_nonce); + assert_eq!(txq.status().pending, 1); + txq.add(tx2, &default_nonce); + assert_eq!(txq.status().future, 1); + + // when + txq.add(tx1, &default_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 3); + assert_eq!(stats.future, 0); + } + + #[test] + fn should_remove_transaction() { + // given + let mut txq2 = TransactionQueue::new(); + let (tx, tx2) = new_txs(U256::from(3)); + txq2.add(tx.clone(), &default_nonce); + txq2.add(tx2.clone(), &default_nonce); + assert_eq!(txq2.status().pending, 1); + assert_eq!(txq2.status().future, 1); + + // when + txq2.remove(&tx); + txq2.remove(&tx2); + + + // then + let stats = txq2.status(); + assert_eq!(stats.pending, 0); + assert_eq!(stats.future, 0); + } + + #[test] + fn should_move_transactions_to_future_if_gap_introduced() { + // given + let mut txq = TransactionQueue::new(); + let (tx, tx2) = new_txs(U256::from(1)); + let tx3 = new_tx(); + txq.add(tx2.clone(), &default_nonce); + assert_eq!(txq.status().future, 1); + txq.add(tx3.clone(), &default_nonce); + txq.add(tx.clone(), &default_nonce); + assert_eq!(txq.status().pending, 3); + + // when + txq.remove(&tx); + + // then + let stats = txq.status(); + assert_eq!(stats.future, 1); + assert_eq!(stats.pending, 1); + } + + #[test] + fn should_clear_queue() { + // given + let mut txq = TransactionQueue::new(); + let (tx, tx2) = new_txs(U256::one()); + + // add + txq.add(tx2.clone(), &default_nonce); + txq.add(tx.clone(), &default_nonce); + let stats = txq.status(); + assert_eq!(stats.pending, 2); + + // when + txq.clear(); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 0); + } + + #[test] + fn should_drop_old_transactions_when_hitting_the_limit() { + // given + let mut txq = TransactionQueue::with_limits(1, 1); + let (tx, tx2) = new_txs(U256::one()); + txq.add(tx.clone(), &default_nonce); + assert_eq!(txq.status().pending, 1); + + // when + txq.add(tx2.clone(), &default_nonce); + + // then + let t = txq.top_transactions(2); + assert_eq!(txq.status().pending, 1); + assert_eq!(t.len(), 1); + assert_eq!(t[0], tx); + } + + #[test] + fn should_limit_future_transactions() { + let mut txq = TransactionQueue::with_limits(10, 1); + let (tx1, tx2) = new_txs(U256::from(4)); + let (tx3, tx4) = new_txs(U256::from(4)); + txq.add(tx1.clone(), &default_nonce); + txq.add(tx3.clone(), &default_nonce); + assert_eq!(txq.status().pending, 2); + + // when + txq.add(tx2.clone(), &default_nonce); + assert_eq!(txq.status().future, 1); + txq.add(tx4.clone(), &default_nonce); + + // then + assert_eq!(txq.status().future, 1); + } + + #[test] + fn should_drop_transactions_with_old_nonces() { + let mut txq = TransactionQueue::new(); + let tx = new_tx(); + let last_nonce = tx.nonce.clone(); + let fetch_last_nonce = |_a: &Address| last_nonce; + + // when + txq.add(tx, &fetch_last_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 0); + assert_eq!(stats.future, 0); + } + + #[test] + fn should_accept_same_transaction_twice() { + // given + let mut txq = TransactionQueue::new(); + let (tx1, tx2) = new_txs(U256::from(1)); + txq.add(tx1.clone(), &default_nonce); + txq.add(tx2.clone(), &default_nonce); + assert_eq!(txq.status().pending, 2); + + // when + txq.remove(&tx1); + assert_eq!(txq.status().future, 1); + txq.add(tx1.clone(), &default_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.pending, 2); + assert_eq!(stats.future, 0); + + } + +} From 30e7ac8d6d4a9f0f6553d48bedd82083266aaa7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 1 Mar 2016 21:54:53 +0100 Subject: [PATCH 18/89] Fixing trivial warnings --- util/src/table.rs | 2 +- util/src/uint.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/util/src/table.rs b/util/src/table.rs index f04a498f8..e41209608 100644 --- a/util/src/table.rs +++ b/util/src/table.rs @@ -111,7 +111,7 @@ impl Table /// /// Returns previous value (if any) pub fn insert(&mut self, row: Row, col: Col, val: Val) -> Option { - self.map.entry(row).or_insert_with(|| HashMap::new()).insert(col, val) + self.map.entry(row).or_insert_with(HashMap::new).insert(col, val) } } diff --git a/util/src/uint.rs b/util/src/uint.rs index 88256d5f2..068456334 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -1982,6 +1982,7 @@ mod tests { #[test] + #[cfg_attr(feature = "dev", allow(cyclomatic_complexity))] fn u256_multi_full_mul() { let result = U256([0, 0, 0, 0]).full_mul(U256([0, 0, 0, 0])); assert_eq!(U512([0, 0, 0, 0, 0, 0, 0, 0]), result); From 083747dc674c63739cdde254ca9731e2fb784343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 1 Mar 2016 22:03:29 +0100 Subject: [PATCH 19/89] Lowering complexity of request_blocks --- sync/src/chain.rs | 159 ++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 77 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 6a7add27f..1d90e2d03 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -249,14 +249,14 @@ impl ChainSync { blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), - mem_used: + mem_used: // TODO: https://github.com/servo/heapsize/pull/50 - // self.downloading_hashes.heap_size_of_children() - //+ self.downloading_bodies.heap_size_of_children() - //+ self.downloading_hashes.heap_size_of_children() - self.headers.heap_size_of_children() - + self.bodies.heap_size_of_children() - + self.peers.heap_size_of_children() + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + self.header_ids.heap_size_of_children(), } } @@ -635,16 +635,7 @@ impl ChainSync { match self.last_imported_block { None => 0, Some(x) => x } } - /// Find some headers or blocks to download for a peer. - fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, ignore_others: bool) { - self.clear_peer_download(peer_id); - - if io.chain().queue_info().is_full() { - self.pause_sync(); - return; - } - - // check to see if we need to download any block bodies first + fn find_block_bodies_hashes_to_request(&self, ignore_others: bool) -> (Vec, Vec) { let mut needed_bodies: Vec = Vec::new(); let mut needed_numbers: Vec = Vec::new(); @@ -664,74 +655,88 @@ impl ChainSync { } } } + (needed_bodies, needed_numbers) + } + + /// Find some headers or blocks to download for a peer. + fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, ignore_others: bool) { + self.clear_peer_download(peer_id); + + if io.chain().queue_info().is_full() { + self.pause_sync(); + return; + } + + // check to see if we need to download any block bodies first + let (needed_bodies, needed_numbers) = self.find_block_bodies_hashes_to_request(ignore_others); if !needed_bodies.is_empty() { let (head, _) = self.headers.range_iter().next().unwrap(); if needed_numbers.first().unwrap() - head > self.max_download_ahead_blocks as BlockNumber { trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading block bodies", peer_id, needed_numbers.first().unwrap(), head); self.request_blocks(io, peer_id, true); - return; + } else { + self.downloading_bodies.extend(needed_numbers.iter()); + replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, needed_numbers); + self.request_bodies(io, peer_id, needed_bodies); + } + return; + } + + // check if need to download headers + let mut start = 0; + if !self.have_common_block { + // download backwards until common block is found 1 header at a time + let chain_info = io.chain().chain_info(); + start = chain_info.best_block_number; + if !self.headers.is_empty() { + start = min(start, self.headers.range_iter().next().unwrap().0 - 1); + } + if start == 0 { + self.have_common_block = true; //reached genesis + self.last_imported_hash = Some(chain_info.genesis_hash); + self.last_imported_block = Some(0); + } + } + if self.have_common_block { + let mut headers: Vec = Vec::new(); + let mut prev = self.current_base_block() + 1; + let head = self.headers.range_iter().next().map(|(h, _)| h); + for (next, ref items) in self.headers.range_iter() { + if !headers.is_empty() { + break; + } + if next <= prev { + prev = next + items.len() as BlockNumber; + continue; + } + let mut block = prev; + while block < next && headers.len() < MAX_HEADERS_TO_REQUEST { + if ignore_others || !self.downloading_headers.contains(&(block as BlockNumber)) { + headers.push(block as BlockNumber); + } + block += 1; + } + prev = next + items.len() as BlockNumber; + } + + if !headers.is_empty() { + start = headers[0]; + if head.is_some() && start > head.unwrap() && start - head.unwrap() > self.max_download_ahead_blocks as BlockNumber { + trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading headers", peer_id, start, head.unwrap()); + self.request_blocks(io, peer_id, true); + return; + } + let count = headers.len(); + self.downloading_headers.extend(headers.iter()); + replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, headers); + assert!(!self.headers.have_item(&start)); + self.request_headers_by_number(io, peer_id, start, count, 0, false); } - self.downloading_bodies.extend(needed_numbers.iter()); - replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, needed_numbers); - self.request_bodies(io, peer_id, needed_bodies); } else { - // check if need to download headers - let mut start = 0; - if !self.have_common_block { - // download backwards until common block is found 1 header at a time - let chain_info = io.chain().chain_info(); - start = chain_info.best_block_number; - if !self.headers.is_empty() { - start = min(start, self.headers.range_iter().next().unwrap().0 - 1); - } - if start == 0 { - self.have_common_block = true; //reached genesis - self.last_imported_hash = Some(chain_info.genesis_hash); - self.last_imported_block = Some(0); - } - } - if self.have_common_block { - let mut headers: Vec = Vec::new(); - let mut prev = self.current_base_block() + 1; - let head = self.headers.range_iter().next().map(|(h, _)| h); - for (next, ref items) in self.headers.range_iter() { - if !headers.is_empty() { - break; - } - if next <= prev { - prev = next + items.len() as BlockNumber; - continue; - } - let mut block = prev; - while block < next && headers.len() < MAX_HEADERS_TO_REQUEST { - if ignore_others || !self.downloading_headers.contains(&(block as BlockNumber)) { - headers.push(block as BlockNumber); - } - block += 1; - } - prev = next + items.len() as BlockNumber; - } - - if !headers.is_empty() { - start = headers[0]; - if head.is_some() && start > head.unwrap() && start - head.unwrap() > self.max_download_ahead_blocks as BlockNumber { - trace!(target: "sync", "{}: Stalled download ({} vs {}), helping with downloading headers", peer_id, start, head.unwrap()); - self.request_blocks(io, peer_id, true); - return; - } - let count = headers.len(); - self.downloading_headers.extend(headers.iter()); - replace(&mut self.peers.get_mut(&peer_id).unwrap().asking_blocks, headers); - assert!(!self.headers.have_item(&start)); - self.request_headers_by_number(io, peer_id, start, count, 0, false); - } - } - else { - // continue search for common block - self.downloading_headers.insert(start); - self.request_headers_by_number(io, peer_id, start, 1, 0, false); - } + // continue search for common block + self.downloading_headers.insert(start); + self.request_headers_by_number(io, peer_id, start, 1, 0, false); } } From 7565625ce042cfc8ce328f9387d8732da48cc4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 1 Mar 2016 22:30:23 +0100 Subject: [PATCH 20/89] Integrating TransactionQueue with client --- Cargo.lock | 19 +++++++++ ethcore/src/client.rs | 7 +++ sync/Cargo.toml | 1 + sync/src/chain.rs | 89 ++++++++++++++++++++++++++++++++++----- sync/src/lib.rs | 14 ++++-- sync/src/tests/helpers.rs | 4 ++ 6 files changed, 120 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e558606eb..845c493fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,6 +125,14 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "deque" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "docopt" version = "0.6.78" @@ -259,6 +267,7 @@ dependencies = [ "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -629,6 +638,16 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.1.54" diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index f2894decb..611561c50 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -123,6 +123,9 @@ pub trait BlockChainClient : Sync + Send { /// Get block total difficulty. fn block_total_difficulty(&self, id: BlockId) -> Option; + /// Get address nonce. + fn nonce(&self, address: &Address) -> U256; + /// Get address code. fn code(&self, address: &Address) -> Option; @@ -445,6 +448,10 @@ impl BlockChainClient for Client { Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) } + fn nonce(&self, address: &Address) -> U256 { + self.state().nonce(address) + } + fn code(&self, address: &Address) -> Option { self.state().code(address) } diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 2ce65ca77..993f07a65 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -17,6 +17,7 @@ time = "0.1.34" rand = "0.3.13" heapsize = "0.3" rustc-serialize = "0.3" +rayon = "0.3.1" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 6a7add27f..fa033813e 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -30,14 +30,17 @@ /// use util::*; +use rayon::prelude::*; use std::mem::{replace}; -use ethcore::views::{HeaderView}; +use ethcore::views::{HeaderView, BlockView}; use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId}; use range_collection::{RangeCollection, ToUsize, FromUsize}; use ethcore::error::*; use ethcore::block::Block; +use ethcore::transaction::SignedTransaction; use io::SyncIo; +use transaction_queue::TransactionQueue; use time; use super::SyncConfig; @@ -209,6 +212,8 @@ pub struct ChainSync { max_download_ahead_blocks: usize, /// Network ID network_id: U256, + /// Transactions Queue + transaction_queue: Mutex, } type RlpResponseResult = Result, PacketDecodeError>; @@ -234,6 +239,7 @@ impl ChainSync { last_send_block_number: 0, max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), network_id: config.network_id, + transaction_queue: Mutex::new(TransactionQueue::new()), } } @@ -249,14 +255,14 @@ impl ChainSync { blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), - mem_used: + mem_used: // TODO: https://github.com/servo/heapsize/pull/50 - // self.downloading_hashes.heap_size_of_children() - //+ self.downloading_bodies.heap_size_of_children() - //+ self.downloading_hashes.heap_size_of_children() - self.headers.heap_size_of_children() - + self.bodies.heap_size_of_children() - + self.peers.heap_size_of_children() + // self.downloading_hashes.heap_size_of_children() + //+ self.downloading_bodies.heap_size_of_children() + //+ self.downloading_hashes.heap_size_of_children() + self.headers.heap_size_of_children() + + self.bodies.heap_size_of_children() + + self.peers.heap_size_of_children() + self.header_ids.heap_size_of_children(), } } @@ -292,6 +298,7 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; + self.transaction_queue.lock().unwrap().clear(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -913,8 +920,16 @@ impl ChainSync { } } /// Called when peer sends us new transactions - fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: PeerId, _r: &UntrustedRlp) -> Result<(), PacketDecodeError> { - Ok(()) + fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + let chain = io.chain(); + let item_count = r.item_count(); + trace!(target: "sync", "{} -> Transactions ({} entries)", peer_id, item_count); + let fetch_latest_nonce = |a : &Address| chain.nonce(a); + for i in 0..item_count { + let tx: SignedTransaction = try!(r.val_at(i)); + self.transaction_queue.lock().unwrap().add(tx, &fetch_latest_nonce); + } + Ok(()) } /// Send Status message @@ -1242,6 +1257,34 @@ impl ChainSync { } self.last_send_block_number = chain.best_block_number; } + + /// called when block is imported to chain, updates transactions queue + pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], bad: &[H256]) { + fn fetch_transactions(chain: &BlockChainClient, hash: &H256) -> Vec { + let block = chain + .block(BlockId::Hash(hash.clone())) + .expect("Expected in-chain blocks."); + let block = BlockView::new(&block); + block.transactions() + }; + + let chain = io.chain(); + let good = good.par_iter().map(|h| fetch_transactions(chain, h)); + let bad = bad.par_iter().map(|h| fetch_transactions(chain, h)); + + good.for_each(|txs| { + let mut transaction_queue = self.transaction_queue.lock().unwrap(); + transaction_queue.remove_all(&txs); + }); + bad.for_each(|txs| { + // populate sender + for tx in &txs { + let _sender = tx.sender(); + } + let mut transaction_queue = self.transaction_queue.lock().unwrap(); + transaction_queue.add_all(txs, |a| chain.nonce(a)); + }); + } } #[cfg(test)] @@ -1571,6 +1614,32 @@ mod tests { assert!(result.is_ok()); } + #[test] + fn should_add_transactions_to_queue() { + // given + let mut client = TestBlockChainClient::new(); + // client.add_blocks(98, BlocksWith::Uncle); + // client.add_blocks(1, BlocksWith::UncleAndTransaction); + // client.add_blocks(1, BlocksWith::Transaction); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); + + let good_blocks = vec![client.block_hash_delta_minus(2)]; + let bad_blocks = vec![client.block_hash_delta_minus(1)]; + + let mut queue = VecDeque::new(); + let io = TestIo::new(&mut client, &mut queue, None); + + // when + sync.chain_new_blocks(&io, &[], &good_blocks); + assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); + sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); + + // then + let status = sync.transaction_queue.lock().unwrap().status(); + assert_eq!(status.pending, 1); + assert_eq!(status.future, 0); + } + #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 74541660d..44f3f02e0 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -54,6 +54,7 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; +extern crate rayon; #[macro_use] extern crate heapsize; @@ -70,8 +71,7 @@ use io::NetSyncIo; mod chain; mod io; mod range_collection; -// TODO [todr] Made public to suppress dead code warnings -pub mod transaction_queue; +mod transaction_queue; #[cfg(test)] mod tests; @@ -153,8 +153,14 @@ impl NetworkProtocolHandler for EthSync { } fn message(&self, io: &NetworkContext, message: &SyncMessage) { - if let SyncMessage::BlockVerified = *message { - self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); + match *message { + SyncMessage::BlockVerified => { + self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); + }, + SyncMessage::NewChainBlocks { ref good, ref bad } => { + let sync_io = NetSyncIo::new(io, self.chain.deref()); + self.sync.write().unwrap().chain_new_blocks(&sync_io, good, bad); + } } } } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index b788e0c2a..302836920 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -105,6 +105,10 @@ impl BlockChainClient for TestBlockChainClient { Some(U256::zero()) } + fn nonce(&self, _address: &Address) -> U256 { + U256::zero() + } + fn code(&self, _address: &Address) -> Option { unimplemented!(); } From c6934431d12af045364bb9256d45305969ccee67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 25 Feb 2016 11:49:12 +0100 Subject: [PATCH 21/89] Adding test for sync.chain_new_blocks. --- sync/src/chain.rs | 33 ++++++++++---------- sync/src/tests/chain.rs | 51 ++++++++++++++++--------------- sync/src/tests/helpers.rs | 57 +++++++++++++++++++++++++++-------- sync/src/transaction_queue.rs | 1 + 4 files changed, 88 insertions(+), 54 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fa033813e..9edab791a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1425,7 +1425,7 @@ mod tests { #[test] fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let io = TestIo::new(&mut client, &mut queue, None); @@ -1438,7 +1438,7 @@ mod tests { #[test] fn calculates_tree_for_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(15, false); + client.add_blocks(15, BlocksWith::Uncle); let start = client.block_hash_delta_minus(4); let end = client.block_hash_delta_minus(2); @@ -1455,7 +1455,7 @@ mod tests { #[test] fn sends_new_hashes_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1475,7 +1475,7 @@ mod tests { #[test] fn sends_latest_block_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1495,7 +1495,7 @@ mod tests { #[test] fn handles_peer_new_block_mallformed() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let block_data = get_dummy_block(11, client.chain_info().best_block_hash); @@ -1513,7 +1513,7 @@ mod tests { #[test] fn handles_peer_new_block() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); @@ -1531,7 +1531,7 @@ mod tests { #[test] fn handles_peer_new_block_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1547,7 +1547,7 @@ mod tests { #[test] fn handles_peer_new_hashes() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1563,7 +1563,7 @@ mod tests { #[test] fn handles_peer_new_hashes_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, false); + client.add_blocks(10, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1581,7 +1581,7 @@ mod tests { #[test] fn hashes_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1600,7 +1600,7 @@ mod tests { #[test] fn block_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let best_hash = client.chain_info().best_block_hash.clone(); @@ -1618,9 +1618,9 @@ mod tests { fn should_add_transactions_to_queue() { // given let mut client = TestBlockChainClient::new(); - // client.add_blocks(98, BlocksWith::Uncle); - // client.add_blocks(1, BlocksWith::UncleAndTransaction); - // client.add_blocks(1, BlocksWith::Transaction); + client.add_blocks(98, BlocksWith::Uncle); + client.add_blocks(1, BlocksWith::UncleAndTransaction); + client.add_blocks(1, BlocksWith::Transaction); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; @@ -1631,6 +1631,7 @@ mod tests { // when sync.chain_new_blocks(&io, &[], &good_blocks); + assert_eq!(sync.transaction_queue.lock().unwrap().status().future, 0); assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); @@ -1643,7 +1644,7 @@ mod tests { #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1667,7 +1668,7 @@ mod tests { #[test] fn returns_requested_block_headers_reverse() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, false); + client.add_blocks(100, BlocksWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index 1dd9a1e78..78663aa16 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -24,8 +24,8 @@ use super::helpers::*; fn two_peers() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); assert_eq!(net.peer(0).chain.blocks.read().unwrap().deref(), net.peer(1).chain.blocks.read().unwrap().deref()); @@ -35,8 +35,8 @@ fn two_peers() { fn status_after_sync() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync(); let status = net.peer(0).sync.status(); assert_eq!(status.state, SyncState::Idle); @@ -45,8 +45,8 @@ fn status_after_sync() { #[test] fn takes_few_steps() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(100, false); - net.peer_mut(2).chain.add_blocks(100, false); + net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(100, BlocksWith::Uncle); let total_steps = net.sync(); assert!(total_steps < 7); } @@ -56,8 +56,9 @@ fn empty_blocks() { ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { - net.peer_mut(1).chain.add_blocks(5, n % 2 == 0); - net.peer_mut(2).chain.add_blocks(5, n % 2 == 0); + let with = if n % 2 == 0 { BlocksWith::Nothing } else { BlocksWith::Uncle }; + net.peer_mut(1).chain.add_blocks(5, with.clone()); + net.peer_mut(2).chain.add_blocks(5, with); } net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); @@ -68,14 +69,14 @@ fn empty_blocks() { fn forked() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(0).chain.add_blocks(300, false); - net.peer_mut(1).chain.add_blocks(300, false); - net.peer_mut(2).chain.add_blocks(300, false); - net.peer_mut(0).chain.add_blocks(100, true); //fork - net.peer_mut(1).chain.add_blocks(200, false); - net.peer_mut(2).chain.add_blocks(200, false); - net.peer_mut(1).chain.add_blocks(100, false); //fork between 1 and 2 - net.peer_mut(2).chain.add_blocks(10, true); + net.peer_mut(0).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(300, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, BlocksWith::Nothing); //fork + net.peer_mut(1).chain.add_blocks(200, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(200, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, BlocksWith::Nothing); // peer 1 has the best chain of 601 blocks let peer1_chain = net.peer(1).chain.numbers.read().unwrap().clone(); net.sync(); @@ -87,8 +88,8 @@ fn forked() { #[test] fn restart() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, false); - net.peer_mut(2).chain.add_blocks(1000, false); + net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); net.sync_steps(8); @@ -109,8 +110,8 @@ fn status_empty() { #[test] fn status_packet() { let mut net = TestNet::new(2); - net.peer_mut(0).chain.add_blocks(100, false); - net.peer_mut(1).chain.add_blocks(1, false); + net.peer_mut(0).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1, BlocksWith::Uncle); net.start(); @@ -123,10 +124,10 @@ fn status_packet() { #[test] fn propagade_hashes() { let mut net = TestNet::new(6); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, false); + net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -149,10 +150,10 @@ fn propagade_hashes() { #[test] fn propagade_blocks() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, false); + net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -164,7 +165,7 @@ fn propagade_blocks() { #[test] fn restart_on_malformed_block() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, false); + net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); net.peer_mut(1).chain.corrupt_block(6); net.sync_steps(10); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 302836920..bb980e3f9 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -22,7 +22,7 @@ use io::SyncIo; use chain::ChainSync; use ::SyncConfig; use ethcore::receipt::Receipt; -use ethcore::transaction::LocalizedTransaction; +use ethcore::transaction::{LocalizedTransaction, Transaction, Action}; use ethcore::filter::Filter; use ethcore::log_entry::LocalizedLogEntry; @@ -34,6 +34,14 @@ pub struct TestBlockChainClient { pub difficulty: RwLock, } +#[derive(Clone)] +pub enum BlocksWith { + Nothing, + Uncle, + Transaction, + UncleAndTransaction +} + impl TestBlockChainClient { pub fn new() -> TestBlockChainClient { @@ -44,30 +52,53 @@ impl TestBlockChainClient { last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), }; - client.add_blocks(1, true); // add genesis block + client.add_blocks(1, BlocksWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } - pub fn add_blocks(&mut self, count: usize, empty: bool) { + pub fn add_blocks(&mut self, count: usize, with: BlocksWith) { let len = self.numbers.read().unwrap().len(); for n in len..(len + count) { let mut header = BlockHeader::new(); header.difficulty = From::from(n); header.parent_hash = self.last_hash.read().unwrap().clone(); header.number = n as BlockNumber; - let mut uncles = RlpStream::new_list(if empty {0} else {1}); - if !empty { - let mut uncle_header = BlockHeader::new(); - uncle_header.difficulty = From::from(n); - uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); - uncle_header.number = n as BlockNumber; - uncles.append(&uncle_header); - header.uncles_hash = uncles.as_raw().sha3(); - } + let uncles = match with { + BlocksWith::Uncle | BlocksWith::UncleAndTransaction => { + let mut uncles = RlpStream::new_list(1); + let mut uncle_header = BlockHeader::new(); + uncle_header.difficulty = From::from(n); + uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); + uncle_header.number = n as BlockNumber; + uncles.append(&uncle_header); + header.uncles_hash = uncles.as_raw().sha3(); + uncles + }, + _ => RlpStream::new_list(0) + }; + let txs = match with { + BlocksWith::Transaction | BlocksWith::UncleAndTransaction => { + let mut txs = RlpStream::new_list(1); + let keypair = KeyPair::create().unwrap(); + let tx = Transaction { + action: Action::Create, + value: U256::from(100), + data: "3331600055".from_hex().unwrap(), + gas: U256::from(100_000), + gas_price: U256::one(), + nonce: U256::one() + }; + let signed_tx = tx.sign(&keypair.secret()); + txs.append(&signed_tx); + txs.out() + }, + _ => rlp::NULL_RLP.to_vec() + }; + let mut rlp = RlpStream::new_list(3); rlp.append(&header); - rlp.append_raw(&rlp::NULL_RLP, 1); + rlp.append_raw(&txs, 1); rlp.append_raw(uncles.as_raw(), 1); self.import_block(rlp.as_raw().to_vec()).unwrap(); } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 341607afe..fa6026477 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -346,6 +346,7 @@ impl TransactionQueue { return; } else if next_nonce > nonce { // Droping transaction + trace!(target: "sync", "Dropping transaction with nonce: {} - expecting: {}", nonce, next_nonce); return; } From 929f44fe4f62910fc0eba95257953b09ccd52b24 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 00:34:38 +0100 Subject: [PATCH 22/89] Tests for Client sealing. --- ethcore/src/engine.rs | 9 +++++++-- ethcore/src/ethereum/ethash.rs | 3 ++- ethcore/src/tests/client.rs | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index a05c78ff5..6806fcce3 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -81,9 +81,14 @@ pub trait Engine : Sync + Send { self.verify_block_basic(header, None).and_then(|_| self.verify_block_unordered(header, None)) } - /// Don't forget to call Super::populateFromParent when subclassing & overriding. + /// Don't forget to call Super::populate_from_parent when subclassing & overriding. // TODO: consider including State in the params. - fn populate_from_parent(&self, _header: &mut Header, _parent: &Header) {} + fn populate_from_parent(&self, header: &mut Header, parent: &Header) { + header.parent_hash = parent.hash; + header.difficulty = parent.difficulty; + header.gas_limit = parent.gas_limit; + header.number = parent.number + 1; + } // TODO: builtin contract routing - to do this properly, it will require removing the built-in configuration-reading logic // from Spec into here and removing the Spec::builtins field. diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 6dabe558f..bf0c1d188 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -144,7 +144,8 @@ impl Engine for Ethash { let difficulty = Ethash::boundary_to_difficulty(&Ethash::from_ethash(quick_get_difficulty( &Ethash::to_ethash(header.bare_hash()), header.nonce().low_u64(), - &Ethash::to_ethash(header.mix_hash()) ))); + &Ethash::to_ethash(header.mix_hash()) + ))); if difficulty < header.difficulty { return Err(From::from(BlockError::InvalidProofOfWork(OutOfBounds { min: Some(header.difficulty), max: None, found: difficulty }))); } diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index 83df81fa2..65c9d7358 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use client::{BlockChainClient, Client, ClientConfig, BlockId}; +use block::IsBlock; use tests::helpers::*; use common::*; use devtools::*; @@ -106,3 +107,22 @@ fn can_collect_garbage() { client.tick(); assert!(client.blockchain_cache_info().blocks < 100 * 1024); } + +#[test] +fn can_mine() { + let dummy_blocks = get_good_dummy_block_seq(2); + let client_result = get_test_client_with_blocks(vec![dummy_blocks[0].clone()]); + let client = client_result.reference(); + let b = client.sealing_block(); + let pow_hash = { + let u = b.lock().unwrap(); + match *u { + Some(ref b) => { + assert_eq!(*b.block().header().parent_hash(), BlockView::new(&dummy_blocks[0]).header_view().sha3()); + b.hash() + } + None => { panic!(); } + } + }; + assert!(client.submit_seal(pow_hash, vec![]).is_ok()); +} \ No newline at end of file From 30c68204377742dbd7cda4bd5ac3114713de5cb1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 00:52:18 +0100 Subject: [PATCH 23/89] Refactor and cleanup. --- ethcore/src/block.rs | 9 +++++---- ethcore/src/engine.rs | 3 +-- ethcore/src/ethereum/ethash.rs | 2 +- rpc/src/v1/impls/web3.rs | 2 +- util/src/misc.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 0ca6e88be..f5788baba 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -178,11 +178,12 @@ impl<'x> OpenBlock<'x> { last_hashes: last_hashes, }; - r.block.base.header.set_number(parent.number() + 1); - r.block.base.header.set_author(author); - r.block.base.header.set_extra_data(extra_data); + r.block.base.header.parent_hash = parent.hash(); + r.block.base.header.number = parent.number + 1; + r.block.base.header.author = author; r.block.base.header.set_timestamp_now(parent.timestamp()); - r.block.base.header.set_parent_hash(parent.hash()); + r.block.base.header.extra_data = extra_data; + r.block.base.header.note_dirty(); engine.populate_from_parent(&mut r.block.base.header, parent); engine.on_new_block(&mut r.block); diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index 6806fcce3..d607ce2e2 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -84,10 +84,9 @@ pub trait Engine : Sync + Send { /// Don't forget to call Super::populate_from_parent when subclassing & overriding. // TODO: consider including State in the params. fn populate_from_parent(&self, header: &mut Header, parent: &Header) { - header.parent_hash = parent.hash; header.difficulty = parent.difficulty; header.gas_limit = parent.gas_limit; - header.number = parent.number + 1; + header.note_dirty(); } // TODO: builtin contract routing - to do this properly, it will require removing the built-in configuration-reading logic diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index bf0c1d188..f9810b964 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -104,7 +104,7 @@ impl Engine for Ethash { max(gas_floor_target, gas_limit - gas_limit / bound_divisor + x!(1) + (header.gas_used * x!(6) / x!(5)) / bound_divisor) } }; - + header.note_dirty(); // info!("ethash: populate_from_parent #{}: difficulty={} and gas_limit={}", header.number, header.difficulty, header.gas_limit); } diff --git a/rpc/src/v1/impls/web3.rs b/rpc/src/v1/impls/web3.rs index 0a237d56b..4d31f73ae 100644 --- a/rpc/src/v1/impls/web3.rs +++ b/rpc/src/v1/impls/web3.rs @@ -31,7 +31,7 @@ impl Web3 for Web3Client { fn client_version(&self, params: Params) -> Result { match params { Params::None => { - Ok(Value::String(version())), + Ok(Value::String(version().to_owned().replace("Parity/", "Parity//"))), } _ => Err(Error::invalid_params()) } diff --git a/util/src/misc.rs b/util/src/misc.rs index 6cd506d02..39ccbf2da 100644 --- a/util/src/misc.rs +++ b/util/src/misc.rs @@ -70,7 +70,7 @@ pub fn contents(name: &str) -> Result { /// Get the standard version string for this software. pub fn version() -> String { - format!("Parity//v{}-{}-{}/{}-{}-{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date().replace("-", ""), Target::arch(), Target::os(), Target::env(), rustc_version::version()) + format!("Parity/v{}-{}-{}/{}-{}-{}/rustc{}", env!("CARGO_PKG_VERSION"), short_sha(), commit_date().replace("-", ""), Target::arch(), Target::os(), Target::env(), rustc_version::version()) } /// Get the standard version data for this software. From 68ba0162792dee1ec350ae2ede8dc254f941a18e Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 04:25:03 +0100 Subject: [PATCH 24/89] improved blockchain generator --- ethcore/src/blockchain/blockchain.rs | 66 ++++---- ethcore/src/blockchain/generator/block.rs | 64 ++++++++ ethcore/src/blockchain/generator/bloom.rs | 35 ++++ ethcore/src/blockchain/generator/complete.rs | 53 ++++++ ethcore/src/blockchain/generator/fork.rs | 42 +++++ .../generators.rs => generator/generator.rs} | 154 +++++++----------- .../blockchain/{helpers => generator}/mod.rs | 9 +- ethcore/src/blockchain/mod.rs | 2 +- 8 files changed, 294 insertions(+), 131 deletions(-) create mode 100644 ethcore/src/blockchain/generator/block.rs create mode 100644 ethcore/src/blockchain/generator/bloom.rs create mode 100644 ethcore/src/blockchain/generator/complete.rs create mode 100644 ethcore/src/blockchain/generator/fork.rs rename ethcore/src/blockchain/{helpers/generators.rs => generator/generator.rs} (51%) rename ethcore/src/blockchain/{helpers => generator}/mod.rs (81%) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 53dcfb62c..10e844192 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -770,14 +770,15 @@ mod tests { use blockchain::{BlockProvider, BlockChain, BlockChainConfig}; use tests::helpers::*; use devtools::*; - use blockchain::helpers::generators::{ChainGenerator, ChainIterator}; + use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer}; use views::BlockView; #[test] fn basic_blockchain_insert() { let mut canon_chain = ChainGenerator::default(); - let genesis = canon_chain.next().unwrap().rlp(); - let first = canon_chain.next().unwrap().rlp(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let first = canon_chain.generate(&mut finalizer).unwrap(); let genesis_hash = BlockView::new(&genesis).header_view().sha3(); let first_hash = BlockView::new(&first).header_view().sha3(); @@ -806,14 +807,12 @@ mod tests { #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { let mut canon_chain = ChainGenerator::default(); - let genesis = canon_chain.next().unwrap().rlp(); - let blocks = canon_chain.clone().take(3).map(|block| block.rlp()).collect::>(); - let fork = canon_chain.skip(2).fork(1).take(1).next().unwrap().rlp(); - - let b1 = blocks[0].clone(); - let b2 = blocks[1].clone(); - let b3a = blocks[2].clone(); - let b3b = fork; + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let b1 = canon_chain.generate(&mut finalizer).unwrap(); + let b2 = canon_chain.generate(&mut finalizer).unwrap(); + let b3b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b3a = canon_chain.generate(&mut finalizer).unwrap(); let genesis_hash = BlockView::new(&genesis).header_view().sha3(); let b1_hash= BlockView::new(&b1).header_view().sha3(); @@ -897,22 +896,24 @@ mod tests { #[test] fn test_reopen_blockchain_db() { - let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); - let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap(); - let genesis_hash = H256::from_str("5716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2").unwrap(); - let b1_hash = H256::from_str("437e51676ff10756fcfee5edd9159fa41dbcb1b2c592850450371cbecd54ee4f").unwrap(); + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let first = canon_chain.generate(&mut finalizer).unwrap(); + let genesis_hash = BlockView::new(&genesis).header_view().sha3(); + let first_hash = BlockView::new(&first).header_view().sha3(); let temp = RandomTempPath::new(); { let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); assert_eq!(bc.best_block_hash(), genesis_hash); - bc.insert_block(&b1, vec![]); - assert_eq!(bc.best_block_hash(), b1_hash); + bc.insert_block(&first, vec![]); + assert_eq!(bc.best_block_hash(), first_hash); } { let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); - assert_eq!(bc.best_block_hash(), b1_hash); + assert_eq!(bc.best_block_hash(), first_hash); } } @@ -976,29 +977,24 @@ mod tests { #[test] fn test_bloom_filter_simple() { - let genesis = "f901fcf901f7a00000000000000000000000000000000000000000000000000000000000000000a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a07dba07d6b448a186e9612e5f737d1c909dce473e53199901a302c00646d523c1a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000080832fefd8808454c98c8142a059262c330941f3fe2a34d16d6e3c7b30d2ceb37c6a0e9a994c494ee1a61d2410885aa4c8bf8e56e264c0c0".from_hex().unwrap(); - - // block b1 (child of genesis) - let b1 = "f90261f901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0cb52de543653d86ccd13ba3ddf8b052525b04231c6884a4db3188a184681d878a0e78628dd45a1f8dc495594d83b76c588a3ee67463260f8b7d4a42f574aeab29aa0e9244cf7503b79c03d3a099e07a80d2dbc77bb0b502d8a89d51ac0d68dd31313b90100000000200000000000000000000000000000000000000000020000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080004000000000000000000000020008302000001832fefd882520884562791e580a051b3ecba4e3f2b49c11d42dd0851ec514b1be3138080f72a2b6e83868275d98f8877671f479c414b47f862f86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ca09e2709d7ec9bbe6b1bbbf0b2088828d14cd5e8642a1fee22dc74bfa89761a7f9a04bd8813dee4be989accdb708b1c2e325a7e9c695a8024e30e89d6c644e424747c0".from_hex().unwrap(); - - // block b2 (child of b1) - let b2 = "f902ccf901f9a04ef46c05763fffc5f7e59f92a7ef438ffccbb578e6e5d0f04e3df8a7fa6c02f6a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000010000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004002832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - - // prepare for fork (b1a, child of genesis) - let b1a = "f902ccf901f9a05716670833ec874362d65fea27a7cd35af5897d275b31a44944113111e4e96d2a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004001832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - - // fork (b2a, child of b1a, with higher total difficulty) - let b2a = "f902ccf901f9a0626b0774a7cbdad7bdce07b87d74b6fa91c1c359d725076215d76348f8399f56a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004002832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - - // fork back :) - let b3 = "f902ccf901f9a0e6cd7250e4c32b33c906aca30280911c560ac67bd0a05fbeb874f99ac7e7e47aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0c70a5dc56146e5ef025e4e5726a6373c6f12fd2f6784093a19ead0a7d17fb292a040645cbce4fd399e7bb9160b4c30c40d7ee616a030d4e18ef0ed3b02bdb65911a086e608555f63628417032a011d107b36427af37d153f0da02ce3f90fdd5e8c08b90100000000000000000000000000000000000000000000000200000008000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302004003832fefd882c0e384562791e880a0e3cc39ff775cc0a32f175995b92e84b729e5c9a3563ff899e3555b908bc21d75887c3cde283f4846a6f8cdf8cb01018304cb2f8080b87e6060604052606e8060106000396000f360606040526000357c010000000000000000000000000000000000000000000000000000000090048063c0406226146037576035565b005b60406004506056565b6040518082815260200191505060405180910390f35b6000600560006000508190555060059050606b565b90561ba05258615c63503c0a600d6994b12ea5750d45b3c69668e2a371b4fbfb9eeff6b8a0a11be762bc90491231274a2945be35a43f23c27775b1ff24dd521702fe15f73ec0".from_hex().unwrap(); - + // TODO: From here let bloom_b1 = H2048::from_str("00000020000000000000000000000000000000000000000002000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000400000000000000000000002000").unwrap(); let bloom_b2 = H2048::from_str("00000000000000000000000000000000000000000000020000001000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); let bloom_ba = H2048::from_str("00000000000000000000000000000000000000000000020000000800000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let mut fork = canon_chain.fork(1); + let mut fork_finalizer = finalizer.fork(); + let b1 = fork.with_bloom(bloom_b1.clone()).generate(&mut fork_finalizer).unwrap(); + let b2 = fork.with_bloom(bloom_b2.clone()).generate(&mut fork_finalizer).unwrap(); + let b3 = fork.with_bloom(bloom_ba.clone()).generate(&mut fork_finalizer).unwrap(); + let b1a = canon_chain.with_bloom(bloom_ba.clone()).generate(&mut finalizer).unwrap(); + let b2a = canon_chain.with_bloom(bloom_ba.clone()).generate(&mut finalizer).unwrap(); + let temp = RandomTempPath::new(); let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); diff --git a/ethcore/src/blockchain/generator/block.rs b/ethcore/src/blockchain/generator/block.rs new file mode 100644 index 000000000..0a3dad399 --- /dev/null +++ b/ethcore/src/blockchain/generator/block.rs @@ -0,0 +1,64 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::rlp::*; +use util::{H256, H2048}; +use util::U256; +use util::bytes::Bytes; +use header::Header; +use transaction::SignedTransaction; + +use super::fork::Forkable; +use super::bloom::WithBloom; +use super::complete::CompleteBlock; + +/// Helper structure, used for encoding blocks. +#[derive(Default)] +pub struct Block { + pub header: Header, + pub transactions: Vec, + pub uncles: Vec
+} + +impl Encodable for Block { + fn rlp_append(&self, s: &mut RlpStream) { + s.begin_list(3); + s.append(&self.header); + s.append(&self.transactions); + s.append(&self.uncles); + } +} + +impl Forkable for Block { + fn fork(mut self, fork_number: usize) -> Self where Self: Sized { + self.header.difficulty = self.header.difficulty - U256::from(fork_number); + self + } +} + +impl WithBloom for Block { + fn with_bloom(mut self, bloom: H2048) -> Self where Self: Sized { + self.header.log_bloom = bloom; + self + } +} + +impl CompleteBlock for Block { + fn complete(mut self, parent_hash: H256) -> Bytes { + self.header.parent_hash = parent_hash; + encode(&self).to_vec() + } +} diff --git a/ethcore/src/blockchain/generator/bloom.rs b/ethcore/src/blockchain/generator/bloom.rs new file mode 100644 index 000000000..f61c9d1ff --- /dev/null +++ b/ethcore/src/blockchain/generator/bloom.rs @@ -0,0 +1,35 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H2048; + +pub trait WithBloom { + fn with_bloom(self, bloom: H2048) -> Self where Self: Sized; +} + +pub struct Bloom<'a, I> where I: 'a { + pub iter: &'a mut I, + pub bloom: H2048, +} + +impl<'a, I> Iterator for Bloom<'a, I> where I: Iterator, ::Item: WithBloom { + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|item| item.with_bloom(self.bloom.clone())) + } +} diff --git a/ethcore/src/blockchain/generator/complete.rs b/ethcore/src/blockchain/generator/complete.rs new file mode 100644 index 000000000..39bd587a0 --- /dev/null +++ b/ethcore/src/blockchain/generator/complete.rs @@ -0,0 +1,53 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::hash::H256; +use util::bytes::Bytes; +use util::sha3::Hashable; +use views::BlockView; + +#[derive(Default, Clone)] +pub struct BlockFinalizer { + parent_hash: H256 +} + +impl BlockFinalizer { + pub fn fork(&self) -> Self { + self.clone() + } +} + +pub trait CompleteBlock { + fn complete(self, parent_hash: H256) -> Bytes; +} + +pub struct Complete<'a, I> where I: 'a { + pub iter: &'a mut I, + pub finalizer: &'a mut BlockFinalizer, +} + +impl<'a, I> Iterator for Complete<'a, I> where I: Iterator, ::Item: CompleteBlock { + type Item = Bytes; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|item| { + let rlp = item.complete(self.finalizer.parent_hash.clone()); + self.finalizer.parent_hash = BlockView::new(&rlp).header_view().sha3(); + rlp + }) + } +} diff --git a/ethcore/src/blockchain/generator/fork.rs b/ethcore/src/blockchain/generator/fork.rs new file mode 100644 index 000000000..f3a4eb267 --- /dev/null +++ b/ethcore/src/blockchain/generator/fork.rs @@ -0,0 +1,42 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +pub trait Forkable { + fn fork(self, fork_number: usize) -> Self where Self: Sized; +} + +pub struct Fork { + pub iter: I, + pub fork_number: usize, +} + +impl Clone for Fork where I: Iterator + Clone { + fn clone(&self) -> Self { + Fork { + iter: self.iter.clone(), + fork_number: self.fork_number + } + } +} + +impl Iterator for Fork where I: Iterator, ::Item: Forkable { + type Item = ::Item; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|item| item.fork(self.fork_number)) + } +} diff --git a/ethcore/src/blockchain/helpers/generators.rs b/ethcore/src/blockchain/generator/generator.rs similarity index 51% rename from ethcore/src/blockchain/helpers/generators.rs rename to ethcore/src/blockchain/generator/generator.rs index a20b0cf79..d73b81733 100644 --- a/ethcore/src/blockchain/helpers/generators.rs +++ b/ethcore/src/blockchain/generator/generator.rs @@ -14,108 +14,52 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use util::rlp::*; -use util::hash::{H256, H2048}; -use util::uint::{U256}; +use util::hash::H2048; +use util::uint::U256; use util::bytes::Bytes; -use header::{BlockNumber, Header}; -use transaction::SignedTransaction; - -pub trait Forkable { - fn fork(self, fork_number: usize) -> Self where Self: Sized; -} - -pub struct Fork { - iter: I, - fork_number: usize, -} - -impl Iterator for Fork where I: Iterator, ::Item: Forkable { - type Item = ::Item; - - #[inline] - fn next(&mut self) -> Option { - self.iter.next().map(|item| item.fork(self.fork_number)) - } -} - -pub trait WithBloom { - fn with_bloom(self, bloom: H2048) -> Self where Self: Sized; -} - -pub struct Bloom { - iter: I, - bloom: H2048, -} - -impl Iterator for Bloom where I: Iterator, ::Item: WithBloom { - type Item = ::Item; - - #[inline] - fn next(&mut self) -> Option { - self.iter.next().map(|item| item.with_bloom(self.bloom.clone())) - } -} +use header::BlockNumber; +use super::fork::Fork; +use super::bloom::Bloom; +use super::complete::{BlockFinalizer, CompleteBlock, Complete}; +use super::block::Block; /// Chain iterator interface. -pub trait ChainIterator: Iterator { +pub trait ChainIterator: Iterator + Sized { /// Should be called to create a fork of current iterator. /// Blocks generated by fork will have lower difficulty than current chain. - fn fork(&mut self, fork_number: usize) -> Fork where Self: Sized; + fn fork(&self, fork_number: usize) -> Fork where Self: Clone; /// Should be called to make every consecutive block have given bloom. - fn with_bloom(&mut self, bloom: H2048) -> Bloom where Self: Sized; + fn with_bloom<'a>(&'a mut self, bloom: H2048) -> Bloom<'a, Self>; + /// Should be called to complete block. Without complete block may have incorrect hash. + fn complete<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Complete<'a, Self>; + /// Completes and generates block. + fn generate<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Option where Self::Item: CompleteBlock; } -impl ChainIterator for I where I: Iterator + Sized + Clone { - fn fork(&mut self, fork_number: usize) -> Fork { +impl ChainIterator for I where I: Iterator + Sized { + fn fork(&self, fork_number: usize) -> Fork where I: Clone { Fork { iter: self.clone(), fork_number: fork_number } } - fn with_bloom(&mut self, bloom: H2048) -> Bloom { + fn with_bloom<'a>(&'a mut self, bloom: H2048) -> Bloom<'a, Self> { Bloom { - iter: self.clone(), + iter: self, bloom: bloom } } -} -/// Helper structure, used for encoding blocks. -#[derive(Default)] -pub struct Block { - header: Header, - transactions: Vec, - uncles: Vec
-} - -impl Block { - pub fn rlp(&self) -> Bytes { - encode(self).to_vec() + fn complete<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Complete<'a, Self> { + Complete { + iter: self, + finalizer: finalizer + } } -} -impl Encodable for Block { - fn rlp_append(&self, s: &mut RlpStream) { - s.begin_list(3); - s.append(&self.header); - s.append(&self.transactions); - s.append(&self.uncles); - } -} - -impl Forkable for Block { - fn fork(mut self, fork_number: usize) -> Self where Self: Sized { - self.header.difficulty = self.header.difficulty - U256::from(fork_number); - self - } -} - -impl WithBloom for Block { - fn with_bloom(mut self, bloom: H2048) -> Self where Self: Sized { - self.header.log_bloom = bloom; - self + fn generate<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Option where ::Item: CompleteBlock { + self.complete(finalizer).next() } } @@ -124,8 +68,6 @@ impl WithBloom for Block { pub struct ChainGenerator { /// Next block number. number: BlockNumber, - /// Next block parent hash. - parent_hash: H256, /// Next block difficulty. difficulty: U256, } @@ -133,7 +75,6 @@ pub struct ChainGenerator { impl ChainGenerator { fn prepare_block(&self) -> Block { let mut block = Block::default(); - block.header.parent_hash = self.parent_hash.clone(); block.header.number = self.number; block.header.difficulty = self.difficulty; block @@ -144,7 +85,6 @@ impl Default for ChainGenerator { fn default() -> Self { ChainGenerator { number: 0, - parent_hash: H256::default(), difficulty: U256::from(1000), } } @@ -156,30 +96,28 @@ impl Iterator for ChainGenerator { fn next(&mut self) -> Option { let block = self.prepare_block(); self.number += 1; - self.parent_hash = block.header.hash(); Some(block) } } - -#[cfg(test)] mod tests { - use util::hash::H256; + use util::hash::{H256, H2048}; use util::sha3::Hashable; use views::BlockView; - use super::{ChainIterator, ChainGenerator}; + use blockchain::generator::{ChainIterator, ChainGenerator, BlockFinalizer}; #[test] fn canon_chain_generator() { let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); - let genesis_rlp = canon_chain.next().unwrap().rlp(); + let genesis_rlp = canon_chain.generate(&mut finalizer).unwrap(); let genesis = BlockView::new(&genesis_rlp); assert_eq!(genesis.header_view().parent_hash(), H256::default()); assert_eq!(genesis.header_view().number(), 0); - let b1_rlp = canon_chain.next().unwrap().rlp(); + let b1_rlp = canon_chain.generate(&mut finalizer).unwrap(); let b1 = BlockView::new(&b1_rlp); assert_eq!(b1.header_view().parent_hash(), genesis.header_view().sha3()); @@ -187,17 +125,45 @@ mod tests { let mut fork_chain = canon_chain.fork(1); - let b2_rlp_fork = fork_chain.next().unwrap().rlp(); + let b2_rlp_fork = fork_chain.generate(&mut finalizer.fork()).unwrap(); let b2_fork = BlockView::new(&b2_rlp_fork); assert_eq!(b2_fork.header_view().parent_hash(), b1.header_view().sha3()); assert_eq!(b2_fork.header_view().number(), 2); - let b2_rlp = canon_chain.next().unwrap().rlp(); + let b2_rlp = canon_chain.generate(&mut finalizer).unwrap(); let b2 = BlockView::new(&b2_rlp); assert_eq!(b2.header_view().parent_hash(), b1.header_view().sha3()); assert_eq!(b2.header_view().number(), 2); assert!(b2.header_view().difficulty() > b2_fork.header_view().difficulty()); } + + #[test] + fn with_bloom_generator() { + let bloom = H2048([0x1; 256]); + let mut gen = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + + let block0_rlp = gen.with_bloom(bloom).generate(&mut finalizer).unwrap(); + let block1_rlp = gen.generate(&mut finalizer).unwrap(); + let block0 = BlockView::new(&block0_rlp); + let block1 = BlockView::new(&block1_rlp); + + assert_eq!(block0.header_view().number(), 0); + assert_eq!(block0.header_view().parent_hash(), H256::default()); + + assert_eq!(block1.header_view().number(), 1); + assert_eq!(block1.header_view().parent_hash(), block0.header_view().sha3()); + + } + + #[test] + fn generate_1000_blocks() { + let generator = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let blocks: Vec<_> = generator.take(1000).complete(&mut finalizer).collect(); + assert_eq!(blocks.len(), 1000); + } } + diff --git a/ethcore/src/blockchain/helpers/mod.rs b/ethcore/src/blockchain/generator/mod.rs similarity index 81% rename from ethcore/src/blockchain/helpers/mod.rs rename to ethcore/src/blockchain/generator/mod.rs index 233f8f1f8..88fdec0e4 100644 --- a/ethcore/src/blockchain/helpers/mod.rs +++ b/ethcore/src/blockchain/generator/mod.rs @@ -14,4 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -pub mod generators; +mod bloom; +mod block; +mod complete; +mod fork; +pub mod generator; + +pub use self::complete::BlockFinalizer; +pub use self::generator::{ChainIterator, ChainGenerator}; diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs index 60a1aeb33..b0679b563 100644 --- a/ethcore/src/blockchain/mod.rs +++ b/ethcore/src/blockchain/mod.rs @@ -24,7 +24,7 @@ mod cache; mod tree_route; mod update; #[cfg(test)] -mod helpers; +mod generator; pub use self::blockchain::{BlockProvider, BlockChain, BlockChainConfig}; pub use self::cache::CacheSize; From 010659cf12b0accaa678232176d60f338f15f1a9 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 04:54:47 +0100 Subject: [PATCH 25/89] added missing , in comment --- ethcore/src/blockchain/generator/generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/generator/generator.rs b/ethcore/src/blockchain/generator/generator.rs index d73b81733..16f2330c2 100644 --- a/ethcore/src/blockchain/generator/generator.rs +++ b/ethcore/src/blockchain/generator/generator.rs @@ -30,7 +30,7 @@ pub trait ChainIterator: Iterator + Sized { fn fork(&self, fork_number: usize) -> Fork where Self: Clone; /// Should be called to make every consecutive block have given bloom. fn with_bloom<'a>(&'a mut self, bloom: H2048) -> Bloom<'a, Self>; - /// Should be called to complete block. Without complete block may have incorrect hash. + /// Should be called to complete block. Without complete, block may have incorrect hash. fn complete<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Complete<'a, Self>; /// Completes and generates block. fn generate<'a>(&'a mut self, finalizer: &'a mut BlockFinalizer) -> Option where Self::Item: CompleteBlock; From 5f97d519673d2ad05d4739a409728ac10f271393 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 05:46:38 +0100 Subject: [PATCH 26/89] renamed poll_indexed to poll_manager --- rpc/src/v1/helpers/mod.rs | 4 ++-- .../{poll_indexer.rs => poll_manager.rs} | 22 +++++++++---------- rpc/src/v1/impls/eth.rs | 8 +++---- 3 files changed, 17 insertions(+), 17 deletions(-) rename rpc/src/v1/helpers/{poll_indexer.rs => poll_manager.rs} (88%) diff --git a/rpc/src/v1/helpers/mod.rs b/rpc/src/v1/helpers/mod.rs index d212d74ab..b1a5c05ba 100644 --- a/rpc/src/v1/helpers/mod.rs +++ b/rpc/src/v1/helpers/mod.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -mod poll_indexer; +mod poll_manager; mod poll_filter; -pub use self::poll_indexer::PollIndexer; +pub use self::poll_manager::PollManager; pub use self::poll_filter::PollFilter; diff --git a/rpc/src/v1/helpers/poll_indexer.rs b/rpc/src/v1/helpers/poll_manager.rs similarity index 88% rename from rpc/src/v1/helpers/poll_indexer.rs rename to rpc/src/v1/helpers/poll_manager.rs index 11cb30935..1e7addb25 100644 --- a/rpc/src/v1/helpers/poll_indexer.rs +++ b/rpc/src/v1/helpers/poll_manager.rs @@ -39,30 +39,30 @@ impl Clone for PollInfo where F: Clone { } /// Indexes all poll requests. -/// +/// /// Lazily garbage collects unused polls info. -pub struct PollIndexer where T: Timer { +pub struct PollManager where T: Timer { polls: TransientHashMap, T>, next_available_id: PollId } -impl PollIndexer { +impl PollManager { /// Creates new instance of indexer. pub fn new() -> Self { - PollIndexer::new_with_timer(Default::default()) + PollManager::new_with_timer(Default::default()) } } -impl PollIndexer where T: Timer { +impl PollManager where T: Timer { pub fn new_with_timer(timer: T) -> Self { - PollIndexer { + PollManager { polls: TransientHashMap::new_with_timer(POLL_LIFETIME, timer), next_available_id: 0 } } - /// Returns id which can be used for new poll. - /// + /// Returns id which can be used for new poll. + /// /// Stores information when last poll happend. pub fn create_poll(&mut self, filter: F, block: BlockNumber) -> PollId { self.polls.prune(); @@ -84,7 +84,7 @@ impl PollIndexer where T: Timer { } /// Returns number of block when last poll happend. - pub fn get_poll(&mut self, id: &PollId) -> Option<&PollInfo> { + pub fn get_poll_info(&mut self, id: &PollId) -> Option<&PollInfo> { self.polls.prune(); self.polls.get(id) } @@ -99,7 +99,7 @@ impl PollIndexer where T: Timer { mod tests { use std::cell::RefCell; use transient_hashmap::Timer; - use v1::helpers::PollIndexer; + use v1::helpers::PollManager; struct TestTimer<'a> { time: &'a RefCell @@ -118,7 +118,7 @@ mod tests { time: &time }; - let mut indexer = PollIndexer::new_with_timer(timer); + let mut indexer = PollManager::new_with_timer(timer); assert_eq!(indexer.create_poll(false, 20), 0); assert_eq!(indexer.create_poll(true, 20), 1); diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 32c143a84..880c45041 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -25,7 +25,7 @@ use ethcore::views::*; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, OptionalValue, Index, Filter, Log}; -use v1::helpers::{PollFilter, PollIndexer}; +use v1::helpers::{PollFilter, PollManager}; /// Eth rpc implementation. pub struct EthClient { @@ -214,7 +214,7 @@ impl Eth for EthClient { /// Eth filter rpc implementation. pub struct EthFilterClient { client: Weak, - polls: Mutex>, + polls: Mutex>, } impl EthFilterClient { @@ -222,7 +222,7 @@ impl EthFilterClient { pub fn new(client: &Arc) -> Self { EthFilterClient { client: Arc::downgrade(client), - polls: Mutex::new(PollIndexer::new()) + polls: Mutex::new(PollManager::new()) } } } @@ -263,7 +263,7 @@ impl EthFilter for EthFilterClient { let client = take_weak!(self.client); from_params::<(Index,)>(params) .and_then(|(index,)| { - let info = self.polls.lock().unwrap().get_poll(&index.value()).cloned(); + let info = self.polls.lock().unwrap().get_poll_info(&index.value()).cloned(); match info { None => Ok(Value::Array(vec![] as Vec)), Some(info) => match info.filter { From 05897ddbb0fd97e69209ef44b72886a5979a7e4c Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 05:50:28 +0100 Subject: [PATCH 27/89] fixed pending transaction mock implementation --- rpc/src/v1/impls/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 880c45041..f885441c5 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -280,7 +280,7 @@ impl EthFilter for EthFilterClient { }, PollFilter::PendingTransaction => { // TODO: fix implementation - to_value(&client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + to_value(&vec![H256::default()]) }, PollFilter::Logs(mut filter) => { filter.from_block = BlockId::Number(info.block_number); From 32074cc420edf2f450e29c89a7ef64a58d2c6d6a Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 05:55:42 +0100 Subject: [PATCH 28/89] fixed compilation issue caused by incorrect merge --- ethcore/src/blockchain/generator/generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/generator/generator.rs b/ethcore/src/blockchain/generator/generator.rs index 16f2330c2..51e6294fc 100644 --- a/ethcore/src/blockchain/generator/generator.rs +++ b/ethcore/src/blockchain/generator/generator.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use util::hash::H2048; -use util::uint::U256; +use util::numbers::U256; use util::bytes::Bytes; use header::BlockNumber; use super::fork::Fork; From 67a1f2065e2611c73e64bbf74c0812b224b64fd7 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 06:00:00 +0100 Subject: [PATCH 29/89] fixed broken master --- ethcore/src/blockchain/helpers/generators.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/helpers/generators.rs b/ethcore/src/blockchain/helpers/generators.rs index a20b0cf79..2fd517e50 100644 --- a/ethcore/src/blockchain/helpers/generators.rs +++ b/ethcore/src/blockchain/helpers/generators.rs @@ -16,7 +16,7 @@ use util::rlp::*; use util::hash::{H256, H2048}; -use util::uint::{U256}; +use util::numbers::{U256}; use util::bytes::Bytes; use header::{BlockNumber, Header}; use transaction::SignedTransaction; From feff1f9e6e8837d1783ff307caa195af9ab63554 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 06:09:27 +0100 Subject: [PATCH 30/89] chainfilter shouldnt exclude to_block from results --- ethcore/src/chainfilter/chainfilter.rs | 24 ++++++++++++------------ ethcore/src/chainfilter/tests.rs | 16 ++++++++-------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ethcore/src/chainfilter/chainfilter.rs b/ethcore/src/chainfilter/chainfilter.rs index fb6df877a..387d750fd 100644 --- a/ethcore/src/chainfilter/chainfilter.rs +++ b/ethcore/src/chainfilter/chainfilter.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Multilevel blockchain bloom filter. -//! +//! //! ```not_run //! extern crate ethcore_util as util; //! extern crate ethcore; @@ -23,33 +23,33 @@ //! use util::sha3::*; //! use util::hash::*; //! use ethcore::chainfilter::*; -//! +//! //! fn main() { //! let (index_size, bloom_levels) = (16, 3); //! let mut cache = MemoryCache::new(); -//! +//! //! let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(); -//! +//! //! // borrow cache for reading inside the scope //! let modified_blooms = { -//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); +//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); //! let block_number = 39; //! let mut bloom = H2048::new(); //! bloom.shift_bloomed(&address.sha3()); //! filter.add_bloom(&bloom, block_number) //! }; -//! +//! //! // number of updated blooms is equal number of levels //! assert_eq!(modified_blooms.len(), bloom_levels as usize); //! //! // lets inserts modified blooms into the cache //! cache.insert_blooms(modified_blooms); -//! +//! //! // borrow cache for another reading operations //! { -//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); +//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); //! let blocks = filter.blocks_with_address(&address, 10, 40); -//! assert_eq!(blocks.len(), 1); +//! assert_eq!(blocks.len(), 1); //! assert_eq!(blocks[0], 39); //! } //! } @@ -71,7 +71,7 @@ pub struct ChainFilter<'a, D> impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource { /// Creates new filter instance. - /// + /// /// Borrows `FilterDataSource` for reading. pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self { ChainFilter { @@ -88,7 +88,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource None => return None, Some(level_bloom) => match level { // if we are on the lowest level - 0 => return match offset < to_block { + 0 => return match offset <= to_block { // take the value if its smaller than to_block true if level_bloom.contains(bloom) => Some(vec![offset]), // return None if it is is equal to to_block @@ -153,7 +153,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource for i in 0..blooms.len() { let index = self.indexer.bloom_index(block_number + i, level); - let new_bloom = { + let new_bloom = { // use new blooms before db blooms where necessary let bloom_at = | index | { result.get(&index).cloned().or_else(|| self.data_source.bloom_at_index(&index)) }; diff --git a/ethcore/src/chainfilter/tests.rs b/ethcore/src/chainfilter/tests.rs index 2baa93e55..08af44720 100644 --- a/ethcore/src/chainfilter/tests.rs +++ b/ethcore/src/chainfilter/tests.rs @@ -22,7 +22,7 @@ use util::sha3::*; use chainfilter::{BloomIndex, FilterDataSource, ChainFilter}; /// In memory cache for blooms. -/// +/// /// Stores all blooms in HashMap, which indexes them by `BloomIndex`. pub struct MemoryCache { blooms: HashMap, @@ -35,7 +35,7 @@ impl MemoryCache { } /// inserts all blooms into cache - /// + /// /// if bloom at given index already exists, overwrites it pub fn insert_blooms(&mut self, blooms: HashMap) { self.blooms.extend(blooms); @@ -81,13 +81,13 @@ fn test_topic_basic_search() { { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_bloom(&to_bloom(&topic), 0, 23); + let blocks = filter.blocks_with_bloom(&to_bloom(&topic), 0, 22); assert_eq!(blocks.len(), 0); } { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_bloom(&to_bloom(&topic), 23, 24); + let blocks = filter.blocks_with_bloom(&to_bloom(&topic), 23, 23); assert_eq!(blocks.len(), 1); assert_eq!(blocks[0], 23); } @@ -144,7 +144,7 @@ fn test_reset_chain_head_simple() { cache.insert_blooms(modified_blooms_3); - + let reset_modified_blooms = { let filter = ChainFilter::new(&cache, index_size, bloom_levels); filter.reset_chain_head(&[to_bloom(&topic_4), to_bloom(&topic_5)], 15, 17) @@ -183,7 +183,7 @@ fn for_each_bloom(bytes: &[u8], mut f: F) where F: FnMut(usize, &H2048) { } fn for_each_log(bytes: &[u8], mut f: F) where F: FnMut(usize, &Address, &[H256]) { - let mut reader = BufReader::new(bytes); + let mut reader = BufReader::new(bytes); let mut line = String::new(); while reader.read_line(&mut line).unwrap() > 0 { { @@ -235,11 +235,11 @@ fn test_chainfilter_real_data_short_searches() { for_each_log(include_bytes!("logs.txt"), | block_number, address, topics | { println!("block_number: {:?}", block_number); let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_bloom(&to_bloom(address), block_number, block_number + 1); + let blocks = filter.blocks_with_bloom(&to_bloom(address), block_number, block_number); assert_eq!(blocks.len(), 1); for (i, topic) in topics.iter().enumerate() { println!("topic: {:?}", i); - let blocks = filter.blocks_with_bloom(&to_bloom(topic), block_number, block_number + 1); + let blocks = filter.blocks_with_bloom(&to_bloom(topic), block_number, block_number); assert_eq!(blocks.len(), 1); } }); From 34b812696b24427f3eb3bbb1c9f67119acf87230 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 2 Mar 2016 08:49:48 +0300 Subject: [PATCH 31/89] remove unused imports --- util/bigint/src/uint.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/util/bigint/src/uint.rs b/util/bigint/src/uint.rs index 4cc0dfc0e..5b67ba41d 100644 --- a/util/bigint/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -40,20 +40,13 @@ use std::fmt; use std::cmp; use std::mem; -use std::ops; -use std::slice; -use std::result; -use std::option; use std::str::{FromStr}; use std::convert::From; use std::hash::{Hash, Hasher}; use std::ops::*; use std::cmp::*; -use std::collections::*; use serde; -use rustc_serialize::json::Json; -use rustc_serialize::base64::FromBase64; use rustc_serialize::hex::{FromHex, FromHexError, ToHex}; From 5dfc3d28491e0e98b54e34a027031e6a54163832 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 2 Mar 2016 13:01:38 +0300 Subject: [PATCH 32/89] resolving path at runtime --- util/cov.sh | 2 +- util/src/keys/geth_import.rs | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/util/cov.sh b/util/cov.sh index 19aa3b892..121f9629d 100755 --- a/util/cov.sh +++ b/util/cov.sh @@ -5,5 +5,5 @@ fi cargo test --no-run || exit $? mkdir -p target/coverage -kcov --exclude-pattern ~/.multirust,rocksdb,secp256k1 --include-pattern src --verify target/coverage target/debug/ethcore_util* +./kcov --exclude-pattern ~/.multirust,rocksdb,secp256k1 --include-pattern src --verify target/coverage target/debug/ethcore_util* xdg-open target/coverage/index.html diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs index cdbd9b213..dbd9f0fe0 100644 --- a/util/src/keys/geth_import.rs +++ b/util/src/keys/geth_import.rs @@ -99,10 +99,20 @@ mod tests { use common::*; use keys::store::SecretStore; + fn test_path() -> &'static str { + match ::std::fs::metadata("res") { + Ok(_) => "res/geth_keystore", + Err(_) => "util/res/geth_keystore" + } + } + + fn test_path_param(param_val: &'static str) -> String { + test_path().to_owned() + param_val + } #[test] fn can_enumerate() { - let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap(); + let keys = enumerate_geth_keys(Path::new(test_path())).unwrap(); assert_eq!(2, keys.len()); } @@ -110,7 +120,7 @@ mod tests { fn can_import() { let temp = ::devtools::RandomTempPath::create_dir(); let mut secret_store = SecretStore::new_in(temp.as_path()); - import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap(); + import_geth_key(&mut secret_store, Path::new(&test_path_param("/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9"))).unwrap(); let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); assert!(key.is_some()); } @@ -119,7 +129,7 @@ mod tests { fn can_import_directory() { let temp = ::devtools::RandomTempPath::create_dir(); let mut secret_store = SecretStore::new_in(temp.as_path()); - import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap(); let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); assert!(key.is_some()); @@ -134,7 +144,7 @@ mod tests { let temp = ::devtools::RandomTempPath::create_dir(); { let mut secret_store = SecretStore::new_in(temp.as_path()); - import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap(); } let key_directory = KeyDirectory::new(&temp.as_path()); @@ -156,7 +166,7 @@ mod tests { let temp = ::devtools::RandomTempPath::create_dir(); let mut secret_store = SecretStore::new_in(temp.as_path()); - import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + import_geth_keys(&mut secret_store, Path::new(test_path())).unwrap(); let val = secret_store.get::(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123"); assert!(val.is_ok()); From 771fbcbd27bd900902f7705bd9d107f129e0ca2b Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 2 Mar 2016 13:02:33 +0300 Subject: [PATCH 33/89] remove redundant modification --- util/cov.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/cov.sh b/util/cov.sh index 121f9629d..19aa3b892 100755 --- a/util/cov.sh +++ b/util/cov.sh @@ -5,5 +5,5 @@ fi cargo test --no-run || exit $? mkdir -p target/coverage -./kcov --exclude-pattern ~/.multirust,rocksdb,secp256k1 --include-pattern src --verify target/coverage target/debug/ethcore_util* +kcov --exclude-pattern ~/.multirust,rocksdb,secp256k1 --include-pattern src --verify target/coverage target/debug/ethcore_util* xdg-open target/coverage/index.html From 28c5f6f9c3acdab23a8b990cad133da6d9381bb6 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 12:32:30 +0100 Subject: [PATCH 34/89] fixed failing test --- rpc/src/v1/helpers/poll_manager.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rpc/src/v1/helpers/poll_manager.rs b/rpc/src/v1/helpers/poll_manager.rs index 1e7addb25..33b98af45 100644 --- a/rpc/src/v1/helpers/poll_manager.rs +++ b/rpc/src/v1/helpers/poll_manager.rs @@ -124,21 +124,21 @@ mod tests { *time.borrow_mut() = 10; indexer.update_poll(&0, 21); - assert_eq!(indexer.get_poll(&0).unwrap().filter, false); - assert_eq!(indexer.get_poll(&0).unwrap().block_number, 21); + assert_eq!(indexer.get_poll_info(&0).unwrap().filter, false); + assert_eq!(indexer.get_poll_info(&0).unwrap().block_number, 21); *time.borrow_mut() = 30; indexer.update_poll(&1, 23); - assert_eq!(indexer.get_poll(&1).unwrap().filter, true); - assert_eq!(indexer.get_poll(&1).unwrap().block_number, 23); + assert_eq!(indexer.get_poll_info(&1).unwrap().filter, true); + assert_eq!(indexer.get_poll_info(&1).unwrap().block_number, 23); *time.borrow_mut() = 75; indexer.update_poll(&0, 30); - assert!(indexer.get_poll(&0).is_none()); - assert_eq!(indexer.get_poll(&1).unwrap().filter, true); - assert_eq!(indexer.get_poll(&1).unwrap().block_number, 23); + assert!(indexer.get_poll_info(&0).is_none()); + assert_eq!(indexer.get_poll_info(&1).unwrap().filter, true); + assert_eq!(indexer.get_poll_info(&1).unwrap().block_number, 23); indexer.remove_poll(&1); - assert!(indexer.get_poll(&1).is_none()); + assert!(indexer.get_poll_info(&1).is_none()); } } From bdf3b197ad8400dc1b9037b49cd695b971ee578b Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 12:42:08 +0100 Subject: [PATCH 35/89] limit serde codegen only to rpc types submodule --- rpc/build.rs | 4 ++-- rpc/src/lib.rs | 33 +++++++++++++++++++++++++++++---- rpc/src/lib.rs.in | 30 ------------------------------ rpc/src/v1/impls/web3.rs | 4 +--- rpc/src/v1/types/mod.rs | 22 ++++------------------ rpc/src/v1/types/mod.rs.in | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 57 deletions(-) delete mode 100644 rpc/src/lib.rs.in create mode 100644 rpc/src/v1/types/mod.rs.in diff --git a/rpc/build.rs b/rpc/build.rs index fe1a55694..b5adeaba1 100644 --- a/rpc/build.rs +++ b/rpc/build.rs @@ -9,8 +9,8 @@ mod inner { pub fn main() { let out_dir = env::var_os("OUT_DIR").unwrap(); - let src = Path::new("src/lib.rs.in"); - let dst = Path::new(&out_dir).join("lib.rs"); + let src = Path::new("src/v1/types/mod.rs.in"); + let dst = Path::new(&out_dir).join("mod.rs"); let mut registry = syntex::Registry::new(); diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 8dd58d0b8..4559f766c 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -28,8 +28,33 @@ extern crate ethcore_util as util; extern crate ethcore; extern crate ethsync; -#[cfg(feature = "serde_macros")] -include!("lib.rs.in"); +use self::jsonrpc_core::{IoHandler, IoDelegate}; -#[cfg(not(feature = "serde_macros"))] -include!(concat!(env!("OUT_DIR"), "/lib.rs")); +pub mod v1; + +/// Http server. +pub struct HttpServer { + handler: IoHandler, + threads: usize +} + +impl HttpServer { + /// Construct new http server object with given number of threads. + pub fn new(threads: usize) -> HttpServer { + HttpServer { + handler: IoHandler::new(), + threads: threads + } + } + + /// Add io delegate. + pub fn add_delegate(&mut self, delegate: IoDelegate) where D: Send + Sync + 'static { + self.handler.add_delegate(delegate); + } + + /// Start server asynchronously in new thread + pub fn start_async(self, addr: &str, cors_domain: &str) { + let server = jsonrpc_http_server::Server::new(self.handler, self.threads); + server.start_async(addr, jsonrpc_http_server::AccessControlAllowOrigin::Value(cors_domain.to_owned())) + } +} diff --git a/rpc/src/lib.rs.in b/rpc/src/lib.rs.in deleted file mode 100644 index e17f2b3bb..000000000 --- a/rpc/src/lib.rs.in +++ /dev/null @@ -1,30 +0,0 @@ -use self::jsonrpc_core::{IoHandler, IoDelegate}; - -pub mod v1; - -/// Http server. -pub struct HttpServer { - handler: IoHandler, - threads: usize -} - -impl HttpServer { - /// Construct new http server object with given number of threads. - pub fn new(threads: usize) -> HttpServer { - HttpServer { - handler: IoHandler::new(), - threads: threads - } - } - - /// Add io delegate. - pub fn add_delegate(&mut self, delegate: IoDelegate) where D: Send + Sync + 'static { - self.handler.add_delegate(delegate); - } - - /// Start server asynchronously in new thread - pub fn start_async(self, addr: &str, cors_domain: &str) { - let server = jsonrpc_http_server::Server::new(self.handler, self.threads); - server.start_async(addr, jsonrpc_http_server::AccessControlAllowOrigin::Value(cors_domain.to_owned())) - } -} diff --git a/rpc/src/v1/impls/web3.rs b/rpc/src/v1/impls/web3.rs index 0a237d56b..1cfe35631 100644 --- a/rpc/src/v1/impls/web3.rs +++ b/rpc/src/v1/impls/web3.rs @@ -30,9 +30,7 @@ impl Web3Client { impl Web3 for Web3Client { fn client_version(&self, params: Params) -> Result { match params { - Params::None => { - Ok(Value::String(version())), - } + Params::None => Ok(Value::String(version())), _ => Err(Error::invalid_params()) } } diff --git a/rpc/src/v1/types/mod.rs b/rpc/src/v1/types/mod.rs index 34c1f1cff..adf9be071 100644 --- a/rpc/src/v1/types/mod.rs +++ b/rpc/src/v1/types/mod.rs @@ -14,22 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -mod block; -mod block_number; -mod bytes; -mod filter; -mod index; -mod log; -mod optionals; -mod sync; -mod transaction; +#[cfg(feature = "serde_macros")] +include!("mod.rs.in"); -pub use self::block::{Block, BlockTransactions}; -pub use self::block_number::BlockNumber; -pub use self::bytes::Bytes; -pub use self::filter::Filter; -pub use self::index::Index; -pub use self::log::Log; -pub use self::optionals::OptionalValue; -pub use self::sync::{SyncStatus, SyncInfo}; -pub use self::transaction::Transaction; +#[cfg(not(feature = "serde_macros"))] +include!(concat!(env!("OUT_DIR"), "/mod.rs")); diff --git a/rpc/src/v1/types/mod.rs.in b/rpc/src/v1/types/mod.rs.in new file mode 100644 index 000000000..34c1f1cff --- /dev/null +++ b/rpc/src/v1/types/mod.rs.in @@ -0,0 +1,35 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +mod block; +mod block_number; +mod bytes; +mod filter; +mod index; +mod log; +mod optionals; +mod sync; +mod transaction; + +pub use self::block::{Block, BlockTransactions}; +pub use self::block_number::BlockNumber; +pub use self::bytes::Bytes; +pub use self::filter::Filter; +pub use self::index::Index; +pub use self::log::Log; +pub use self::optionals::OptionalValue; +pub use self::sync::{SyncStatus, SyncInfo}; +pub use self::transaction::Transaction; From 3309959139cd8308367e83cf80d3639866beb268 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 12:57:34 +0100 Subject: [PATCH 36/89] Additional check to ancient enactments. --- ethcore/src/client.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index f2894decb..803f5c9db 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -257,6 +257,14 @@ impl Client { let engine = self.engine.deref().deref(); let header = &block.header; + // Check the block isn't so old we won't be able to enact it. + let best_block_number = self.chain.read().unwrap().best_block_number(); + debug!("Best: {}, importing: {}", best_block_number, header.number()); + if header.number() <= best_block_number - HISTORY { + warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number); + return Err(()); + } + // Verify Block Family let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); if let Err(e) = verify_family_result { From 041cfda80badaf98f4828121a28fabae7591abd1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 2 Mar 2016 13:21:33 +0100 Subject: [PATCH 37/89] Improved journaldb logging --- util/src/journaldb.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index cfcff6ea4..5e6ca47c2 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -121,6 +121,7 @@ impl JournalDB { // and the key is safe to delete. // record new commit's details. + debug!("commit: #{} ({}), end era: {:?}", now, id, end); let batch = DBTransaction::new(); let mut counters = self.counters.write().unwrap(); { @@ -167,15 +168,18 @@ impl JournalDB { &last })) { let rlp = Rlp::new(&rlp_data); - let inserts: Vec = rlp.val_at(1); + let mut inserts: Vec = rlp.val_at(1); JournalDB::decrease_counters(&inserts, &mut counters); // Collect keys to be removed. These are removed keys for canonical block, inserted for non-canonical if canon_id == rlp.val_at(0) { - to_remove.extend(rlp.at(2).iter().map(|r| r.as_val::())); + let mut canon_deletes: Vec = rlp.val_at(2); + trace!("Purging nodes deleted from canon: {:?}", canon_deletes); + to_remove.append(&mut canon_deletes); canon_inserts = inserts; } else { - to_remove.extend(inserts); + trace!("Purging nodes inserted in non-canon: {:?}", inserts); + to_remove.append(&mut inserts); } try!(batch.delete(&last)); index += 1; @@ -188,7 +192,7 @@ impl JournalDB { try!(batch.delete(&h)); deletes += 1; } - trace!("JournalDB: delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); + debug!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); } // Commit overlay insertions @@ -209,7 +213,7 @@ impl JournalDB { } try!(self.backing.write(batch)); - trace!("JournalDB::commit() deleted {} nodes", deletes); + debug!("commit: Deleted {} nodes", deletes); Ok(ret) } @@ -258,7 +262,7 @@ impl JournalDB { era -= 1; } } - trace!("Recovered {} counters", res.len()); + debug!("Recovered {} counters", res.len()); res } } From 47b47293323074f812a713477e435e18219c971f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 13:28:51 +0100 Subject: [PATCH 38/89] Fix check. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 268d99229..bf6f4bd71 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -268,7 +268,7 @@ impl Client { // Check the block isn't so old we won't be able to enact it. let best_block_number = self.chain.read().unwrap().best_block_number(); debug!("Best: {}, importing: {}", best_block_number, header.number()); - if header.number() <= best_block_number - HISTORY { + if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY { warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number); return Err(()); } From 1ae573bf21cc183bbffe06e1c2c84bc8c17c7e85 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 13:40:11 +0100 Subject: [PATCH 39/89] Fix install-deps.sh. --- install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-deps.sh b/install-deps.sh index 6cc7de002..7a5db1003 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -569,7 +569,7 @@ function run_installer() if [[ $isSudo == false ]]; then apt-get install -q -y sudo fi - curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes + curl -sf https://raw.githubusercontent.com/brson/multirust/master/quick-install.sh | sudo sh -s -- --yes echo fi From 190468e1f85175b72f1f460dd5810e7ae4a62da6 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 14:03:43 +0100 Subject: [PATCH 40/89] fixed trailing , --- rpc/src/v1/helpers/poll_manager.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rpc/src/v1/helpers/poll_manager.rs b/rpc/src/v1/helpers/poll_manager.rs index 33b98af45..36a6352c2 100644 --- a/rpc/src/v1/helpers/poll_manager.rs +++ b/rpc/src/v1/helpers/poll_manager.rs @@ -57,7 +57,7 @@ impl PollManager where T: Timer { pub fn new_with_timer(timer: T) -> Self { PollManager { polls: TransientHashMap::new_with_timer(POLL_LIFETIME, timer), - next_available_id: 0 + next_available_id: 0, } } @@ -70,7 +70,7 @@ impl PollManager where T: Timer { self.next_available_id += 1; self.polls.insert(id, PollInfo { filter: filter, - block_number: block + block_number: block, }); id } @@ -102,7 +102,7 @@ mod tests { use v1::helpers::PollManager; struct TestTimer<'a> { - time: &'a RefCell + time: &'a RefCell, } impl<'a> Timer for TestTimer<'a> { @@ -115,7 +115,7 @@ mod tests { fn test_poll_indexer() { let time = RefCell::new(0); let timer = TestTimer { - time: &time + time: &time, }; let mut indexer = PollManager::new_with_timer(timer); From b1a62575c9fe86ebe8299c8f6238892fedfeb6c0 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Mar 2016 14:09:44 +0100 Subject: [PATCH 41/89] fixed PollFilter::PendingTransaction returning empty hash --- rpc/src/v1/impls/eth.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 2baaf5225..2313d5114 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -323,8 +323,8 @@ impl EthFilter for EthFilterClient { to_value(&hashes) }, PollFilter::PendingTransaction => { - // TODO: fix implementation - to_value(&vec![H256::default()]) + // TODO: fix implementation once TransactionQueue is merged + to_value(&vec![] as &Vec) }, PollFilter::Logs(mut filter) => { filter.from_block = BlockId::Number(info.block_number); From 02aad03f9204acfb51ec65b52c2d35395ee76197 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 2 Mar 2016 17:06:53 +0300 Subject: [PATCH 42/89] helpers --- ethcore/src/tests/client.rs | 19 +++++++++++++ ethcore/src/tests/helpers.rs | 52 +++++++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index 83df81fa2..e2671d63e 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -106,3 +106,22 @@ fn can_collect_garbage() { client.tick(); assert!(client.blockchain_cache_info().blocks < 100 * 1024); } + +#[test] +fn can_handle_long_fork() { + let client_result = generate_dummy_client(1200); + let client = client_result.reference(); + for _ in 0..10 { + client.import_verified_blocks(&IoChannel::disconnected()); + } + assert_eq!(1200, client.chain_info().best_block_number); + + push_blocks_to_client(client, 45, 1201, 800); + push_blocks_to_client(client, 49, 1201, 800); + push_blocks_to_client(client, 53, 1201, 600); + + for _ in 0..20 { + client.import_verified_blocks(&IoChannel::disconnected()); + } + assert_eq!(2000, client.chain_info().best_block_number); +} diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 44ad667b9..bb9a44614 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -156,10 +156,9 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult rolling_block_number = rolling_block_number + 1; rolling_timestamp = rolling_timestamp + 10; - if let Err(_) = client.import_block(create_test_block(&header)) { - panic!("error importing block which is valid by definition"); + if let Err(e) = client.import_block(create_test_block(&header)) { + panic!("error importing block which is valid by definition: {:?}", e); } - } client.flush_queue(); client.import_verified_blocks(&IoChannel::disconnected()); @@ -170,6 +169,34 @@ pub fn generate_dummy_client(block_number: u32) -> GuardedTempResult } } +pub fn push_blocks_to_client(client: &Arc, timestamp_salt: u64, starting_number: usize, block_number: usize) { + let test_spec = get_test_spec(); + let test_engine = test_spec.to_engine().unwrap(); + let state_root = test_engine.spec().genesis_header().state_root; + let mut rolling_hash = client.chain_info().best_block_hash; + let mut rolling_block_number = starting_number as u64; + let mut rolling_timestamp = timestamp_salt + starting_number as u64 * 10; + + for _ in 0..block_number { + let mut header = Header::new(); + + header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap()); + header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap()); + header.timestamp = rolling_timestamp; + header.number = rolling_block_number; + header.parent_hash = rolling_hash; + header.state_root = state_root.clone(); + + rolling_hash = header.hash(); + rolling_block_number = rolling_block_number + 1; + rolling_timestamp = rolling_timestamp + 10; + + if let Err(e) = client.import_block(create_test_block(&header)) { + panic!("error importing block which is valid by definition: {:?}", e); + } + } +} + pub fn get_test_client_with_blocks(blocks: Vec) -> GuardedTempResult> { let dir = RandomTempPath::new(); let client = Client::new(ClientConfig::default(), get_test_spec(), dir.as_path(), IoChannel::disconnected()).unwrap(); @@ -253,18 +280,29 @@ pub fn get_temp_state_in(path: &Path) -> State { pub fn get_good_dummy_block_seq(count: usize) -> Vec { let test_spec = get_test_spec(); let test_engine = test_spec.to_engine().unwrap(); - let mut parent = test_engine.spec().genesis_header().hash(); + get_good_dummy_block_fork_seq(1, count, &test_engine.spec().genesis_header().hash()) +} + +pub fn get_good_dummy_block_fork_seq(start_number: usize, count: usize, parent_hash: &H256) -> Vec { + let test_spec = get_test_spec(); + let test_engine = test_spec.to_engine().unwrap(); + let mut rolling_timestamp = start_number as u64 * 10; + let mut parent = *parent_hash; let mut r = Vec::new(); - for i in 1 .. count + 1 { + for i in start_number .. start_number + count + 1 { let mut block_header = Header::new(); block_header.gas_limit = decode(test_engine.spec().engine_params.get("minGasLimit").unwrap()); - block_header.difficulty = decode(test_engine.spec().engine_params.get("minimumDifficulty").unwrap()); - block_header.timestamp = i as u64; + block_header.difficulty = U256::from(i).mul(U256([0, 1, 0, 0])); + block_header.timestamp = rolling_timestamp; block_header.number = i as u64; block_header.parent_hash = parent; block_header.state_root = test_engine.spec().genesis_header().state_root; + parent = block_header.hash(); + rolling_timestamp = rolling_timestamp + 10; + r.push(create_test_block(&block_header)); + } r } From 3c7814c8ac6882e3c9fe8b54e10c7ee2e14e5518 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 15:43:48 +0100 Subject: [PATCH 43/89] Remove debug line. --- ethcore/src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index bf6f4bd71..cc5e23869 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -267,7 +267,6 @@ impl Client { // Check the block isn't so old we won't be able to enact it. let best_block_number = self.chain.read().unwrap().best_block_number(); - debug!("Best: {}, importing: {}", best_block_number, header.number()); if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY { warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number); return Err(()); From c75737bcf0318a51f7bc19ab23d78e630481adf6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 17:04:44 +0100 Subject: [PATCH 44/89] Add ancestry iterator. --- ethcore/src/blockchain/blockchain.rs | 52 ++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 185bcaad3..6e5d92a45 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -227,6 +227,23 @@ impl BlockProvider for BlockChain { const COLLECTION_QUEUE_SIZE: usize = 8; +pub struct AncestryIter<'a> { + current: H256, + chain: &'a BlockChain, +} +impl<'a> Iterator for AncestryIter<'a> { + type Item = H256; + fn next(&mut self) -> Option { + if self.current.is_zero() { + Option::None + } else { + let n = self.chain.block_details(&self.current).unwrap().parent; + self.current = n; + Some(self.current.clone()) + } + } +} + impl BlockChain { /// Create new instance of blockchain from given Genesis pub fn new(config: BlockChainConfig, genesis: &[u8], path: &Path) -> BlockChain { @@ -473,9 +490,40 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } + pub fn ancestry_iter(&self, first: H256) -> AncestryIter { + AncestryIter { + current: first, + chain: &self, + } + } + /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, _parent: &H256) -> Vec
{ - // TODO + pub fn find_uncle_headers(&self, parent: &H256) -> Vec
{ + let uncle_generations = 6usize; +/* + { + // Find great-uncles (or second-cousins or whatever they are) - children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. + clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); + h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); + auto p = m_previousBlock.parentHash(); + for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parent) + { + auto us = _bc.details(p).children; + assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! + for (auto const& u: us) + if (!excluded.count(u)) // ignore any uncles/mainline blocks that we know about. + { + uncleBlockHeaders.push_back(_bc.info(u)); + unclesData.appendRaw(_bc.headerData(u)); + ++unclesCount; + if (unclesCount == 2) + break; + } + } + } +*/ + let _excluded = self.ancestry_iter(parent.clone()).take(uncle_generations).collect::>(); + Vec::new() } From 671965d44fdf60547d85cd0689f62da46321ea4d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 17:31:42 +0100 Subject: [PATCH 45/89] Test for ancestry. --- ethcore/src/blockchain/blockchain.rs | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6e5d92a45..d40a34b0c 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -231,15 +231,16 @@ pub struct AncestryIter<'a> { current: H256, chain: &'a BlockChain, } + impl<'a> Iterator for AncestryIter<'a> { type Item = H256; fn next(&mut self) -> Option { if self.current.is_zero() { Option::None } else { - let n = self.chain.block_details(&self.current).unwrap().parent; - self.current = n; - Some(self.current.clone()) + let mut n = self.chain.block_details(&self.current).unwrap().parent; + mem::swap(&mut self.current, &mut n); + Some(n) } } } @@ -857,6 +858,28 @@ mod tests { assert_eq!(bc.block_hash(2), None); } + #[test] + fn check_ancestry_iter() { + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let genesis_hash = BlockView::new(&genesis).header_view().sha3(); + + let temp = RandomTempPath::new(); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); + + let mut block_hashes = vec![genesis_hash.clone()]; + for _ in 0..10 { + let block = canon_chain.generate(&mut finalizer).unwrap(); + block_hashes.push(BlockView::new(&block).header_view().sha3()); + bc.insert_block(&block, vec![]); + } + + block_hashes.reverse(); + + assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).collect::>(), block_hashes) + } + #[test] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { From 42df98450c3c2edcf9c1898d20c23cb012928b32 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 18:05:47 +0100 Subject: [PATCH 46/89] Include uncles in exclused. --- ethcore/src/blockchain/blockchain.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index d40a34b0c..6779051e2 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -78,7 +78,7 @@ pub trait BlockProvider { } /// Get a list of uncles for a given block. - /// Returns None if block deos not exist. + /// Returns None if block does not exist. fn uncles(&self, hash: &H256) -> Option> { self.block(hash).map(|bytes| BlockView::new(&bytes).uncles()) } @@ -491,7 +491,7 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } - pub fn ancestry_iter(&self, first: H256) -> AncestryIter { + pub fn ancestry_iter(&self, first: H256) -> Option { AncestryIter { current: first, chain: &self, @@ -503,7 +503,8 @@ impl BlockChain { let uncle_generations = 6usize; /* { - // Find great-uncles (or second-cousins or whatever they are) - children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. + // Find great-uncles (or second-cousins or whatever they are) - + // children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); auto p = m_previousBlock.parentHash(); @@ -523,7 +524,11 @@ impl BlockChain { } } */ - let _excluded = self.ancestry_iter(parent.clone()).take(uncle_generations).collect::>(); + let _excluded = self + .ancestry_iter(parent.clone()) + .take(uncle_generations) + .flat_map(|h| self.uncle_hashes(&h).iter().chain(&[h])) + .collect::>(); Vec::new() } From 877270c35fe68adb967a5d0714d5c3e15eaf91fc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 18:32:54 +0100 Subject: [PATCH 47/89] Fixes. --- ethcore/src/blockchain/blockchain.rs | 31 +++++++++++++++++----------- ethcore/src/client.rs | 4 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6779051e2..6b0939df5 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -491,15 +491,20 @@ impl BlockChain { self.extras_db.write(batch).unwrap(); } + /// Iterator that lists `first` and then all of `first`'s ancestors, by hash. pub fn ancestry_iter(&self, first: H256) -> Option { - AncestryIter { - current: first, - chain: &self, + if self.is_known(&first) { + Some(AncestryIter { + current: first, + chain: &self, + }) + } else { + None } } /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, parent: &H256) -> Vec
{ + pub fn find_uncle_headers(&self, parent: &H256) -> Option> { let uncle_generations = 6usize; /* { @@ -524,13 +529,15 @@ impl BlockChain { } } */ - let _excluded = self - .ancestry_iter(parent.clone()) - .take(uncle_generations) - .flat_map(|h| self.uncle_hashes(&h).iter().chain(&[h])) - .collect::>(); - - Vec::new() + if !self.is_known(parent) { return None; } + let mut _excluded = HashSet::new(); + for a in self.ancestry_iter(parent.clone()).unwrap().take(uncle_generations) { + for u in self.uncle_hashes(&a).unwrap().into_iter() { + _excluded.insert(u); + } + _excluded.insert(a); + } + None } /// Get inserted block info which is critical to preapre extras updates. @@ -882,7 +889,7 @@ mod tests { block_hashes.reverse(); - assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).collect::>(), block_hashes) + assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).unwrap().collect::>(), block_hashes) } #[test] diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index cc5e23869..105090580 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -200,7 +200,7 @@ pub struct Client { extra_data: RwLock, } -const HISTORY: u64 = 1000; +const HISTORY: u64 = 2000000; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { @@ -457,7 +457,7 @@ impl Client { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h).into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. From 039c0056bc0f58bfe33860c7a4aab7b9aff57bbc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 19:38:00 +0100 Subject: [PATCH 48/89] Uncle inclusion in block authoring. Still need tests. --- ethcore/src/blockchain/blockchain.rs | 46 +++++++++------------------- ethcore/src/client.rs | 2 +- ethcore/src/engine.rs | 2 ++ ethcore/src/verification.rs | 4 +-- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 6b0939df5..4cf2d96c5 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -503,41 +503,23 @@ impl BlockChain { } } - /// Given a block's `parent`, find every block header which represents a valid uncle. - pub fn find_uncle_headers(&self, parent: &H256) -> Option> { - let uncle_generations = 6usize; -/* - { - // Find great-uncles (or second-cousins or whatever they are) - - // children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations. - clog(StateDetail) << "Checking " << m_previousBlock.hash() << ", parent=" << m_previousBlock.parentHash(); - h256Hash excluded = _bc.allKinFrom(m_currentBlock.parentHash(), 6); - auto p = m_previousBlock.parentHash(); - for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash() && unclesCount < 2; ++gen, p = _bc.details(p).parent) - { - auto us = _bc.details(p).children; - assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent! - for (auto const& u: us) - if (!excluded.count(u)) // ignore any uncles/mainline blocks that we know about. - { - uncleBlockHeaders.push_back(_bc.info(u)); - unclesData.appendRaw(_bc.headerData(u)); - ++unclesCount; - if (unclesCount == 2) - break; - } - } - } -*/ + /// Given a block's `parent`, find every block header which represents a valid possible uncle. + pub fn find_uncle_headers(&self, parent: &H256, uncle_generations: usize) -> Option> { if !self.is_known(parent) { return None; } - let mut _excluded = HashSet::new(); + + let mut excluded = HashSet::new(); for a in self.ancestry_iter(parent.clone()).unwrap().take(uncle_generations) { - for u in self.uncle_hashes(&a).unwrap().into_iter() { - _excluded.insert(u); - } - _excluded.insert(a); + excluded.extend(self.uncle_hashes(&a).unwrap().into_iter()); + excluded.insert(a); } - None + + let mut ret = Vec::new(); + for a in self.ancestry_iter(parent.clone()).unwrap().skip(1).take(uncle_generations) { + ret.extend(self.block_details(&a).unwrap().children.iter() + .filter_map(|h| if excluded.contains(h) { None } else { self.block_header(h) }) + ); + } + Some(ret) } /// Get inserted block info which is critical to preapre extras updates. diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 105090580..4ddb40477 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -457,7 +457,7 @@ impl Client { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. diff --git a/ethcore/src/engine.rs b/ethcore/src/engine.rs index d607ce2e2..83e1986fd 100644 --- a/ethcore/src/engine.rs +++ b/ethcore/src/engine.rs @@ -47,6 +47,8 @@ pub trait Engine : Sync + Send { fn maximum_extra_data_size(&self) -> usize { decode(&self.spec().engine_params.get("maximumExtraDataSize").unwrap()) } /// Maximum number of uncles a block is allowed to declare. fn maximum_uncle_count(&self) -> usize { 2 } + /// The number of generations back that uncles can be. + fn maximum_uncle_age(&self) -> usize { 6 } /// The nonce with which accounts begin. fn account_start_nonce(&self) -> U256 { decode(&self.spec().engine_params.get("accountStartNonce").unwrap()) } diff --git a/ethcore/src/verification.rs b/ethcore/src/verification.rs index f52e2e1e4..23b850f55 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification.rs @@ -94,7 +94,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, b excluded.insert(header.hash()); let mut hash = header.parent_hash.clone(); excluded.insert(hash.clone()); - for _ in 0..6 { + for _ in 0..engine.maximum_uncle_age() { match bc.block_details(&hash) { Some(details) => { excluded.insert(details.parent.clone()); @@ -121,7 +121,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, b // (8 Invalid) let depth = if header.number > uncle.number { header.number - uncle.number } else { 0 }; - if depth > 6 { + if depth > engine.maximum_uncle_age() as u64 { return Err(From::from(BlockError::UncleTooOld(OutOfBounds { min: Some(header.number - depth), max: Some(header.number - 1), found: uncle.number }))); } else if depth < 1 { From fc9999fb059576cb040e3d0d8e2526f693412802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 2 Mar 2016 21:26:48 +0100 Subject: [PATCH 49/89] Changing uint to numbers --- sync/src/transaction_queue.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 341607afe..f2e15f955 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -21,7 +21,7 @@ use std::vec::Vec; use std::cmp::{Ordering}; use std::collections::{HashMap, BTreeSet}; -use util::uint::{Uint, U256}; +use util::numbers::{Uint, U256}; use util::hash::{Address}; use util::table::*; use ethcore::transaction::*; @@ -373,7 +373,7 @@ mod test { use self::rustc_serialize::hex::FromHex; use util::crypto::KeyPair; - use util::uint::{U256, Uint}; + use util::numbers::{U256, Uint}; use util::hash::{Address}; use ethcore::transaction::*; use super::*; From 6933bb971ba8947ead271ec8825452fc3fd7ae20 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Mar 2016 23:41:15 +0100 Subject: [PATCH 50/89] Test. --- ethcore/src/blockchain/blockchain.rs | 38 ++++++++++++++++++++++++++++ ethcore/src/header.rs | 21 ++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 4cf2d96c5..394c16e85 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -874,6 +874,44 @@ mod tests { assert_eq!(bc.ancestry_iter(block_hashes[0].clone()).unwrap().collect::>(), block_hashes) } + #[test] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] + fn test_find_uncles() { + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let b1b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b1a = canon_chain.generate(&mut finalizer).unwrap(); + let b2b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b2a = canon_chain.generate(&mut finalizer).unwrap(); + let b3b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b3a = canon_chain.generate(&mut finalizer).unwrap(); + let b4b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b4a = canon_chain.generate(&mut finalizer).unwrap(); + let b5b = canon_chain.fork(1).generate(&mut finalizer.fork()).unwrap(); + let b5a = canon_chain.generate(&mut finalizer).unwrap(); + + let temp = RandomTempPath::new(); + let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); + bc.insert_block(&b1a, vec![]); + bc.insert_block(&b1b, vec![]); + bc.insert_block(&b2a, vec![]); + bc.insert_block(&b2b, vec![]); + bc.insert_block(&b3a, vec![]); + bc.insert_block(&b3b, vec![]); + bc.insert_block(&b4a, vec![]); + bc.insert_block(&b4b, vec![]); + bc.insert_block(&b5a, vec![]); + bc.insert_block(&b5b, vec![]); + + assert_eq!( + [&b4b, &b3b, &b2b].iter().map(|b| BlockView::new(b).header()).collect::>(), + bc.find_uncle_headers(&BlockView::new(&b4a).header_view().sha3(), 3).unwrap() + ); + + // TODO: insert block that already includes one of them as an uncle to check it's not allowed. + } + #[test] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn test_small_fork() { diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index cc02d84db..1e1a54d57 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -29,7 +29,7 @@ pub type BlockNumber = u64; /// which is non-specific. /// /// Doesn't do all that much on its own. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Eq)] pub struct Header { // TODO: make all private. /// Parent hash. @@ -70,6 +70,25 @@ pub struct Header { pub bare_hash: RefCell>, } +impl PartialEq for Header { + fn eq(&self, c: &Header) -> bool { + self.parent_hash == c.parent_hash && + self.timestamp == c.timestamp && + self.number == c.number && + self.author == c.author && + self.transactions_root == c.transactions_root && + self.uncles_hash == c.uncles_hash && + self.extra_data == c.extra_data && + self.state_root == c.state_root && + self.receipts_root == c.receipts_root && + self.log_bloom == c.log_bloom && + self.gas_used == c.gas_used && + self.gas_limit == c.gas_limit && + self.difficulty == c.difficulty && + self.seal == c.seal + } +} + impl Default for Header { fn default() -> Self { Header { From 3daa4c6497f48915ad3d156836c24638d1592cbc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 11:39:00 +0100 Subject: [PATCH 51/89] Fix max uncles. --- ethcore/src/block.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index f5788baba..d05ce51b9 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -220,7 +220,7 @@ impl<'x> OpenBlock<'x> { /// NOTE Will check chain constraints and the uncle number but will NOT check /// that the header itself is actually valid. pub fn push_uncle(&mut self, valid_uncle_header: Header) -> Result<(), BlockError> { - if self.block.base.uncles.len() >= self.engine.maximum_uncle_count() { + if self.block.base.uncles.len() > self.engine.maximum_uncle_count() { return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len()})); } // TODO: check number From df77f51bccf1cd396154207e0084147b26bf9f93 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 11:47:24 +0100 Subject: [PATCH 52/89] History to 30 to pass tests. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 4ddb40477..bcddec85a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -200,7 +200,7 @@ pub struct Client { extra_data: RwLock, } -const HISTORY: u64 = 2000000; +const HISTORY: u64 = 30; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { From dadc2a96eafac3a72bef19d84f40357adbcf6032 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 12:39:19 +0100 Subject: [PATCH 53/89] shrink_to_fit after removing hashes. --- ethcore/src/blockchain/blockchain.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 185bcaad3..eac6da6a2 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -759,6 +759,14 @@ impl BlockChain { // TODO: handle block_hashes properly. block_hashes.clear(); + + blocks.shrink_to_fit(); + block_details.shrink_to_fit(); + block_hashes.shrink_to_fit(); + transaction_addresses.shrink_to_fit(); + block_logs.shrink_to_fit(); + blocks_blooms.shrink_to_fit(); + block_receipts.shrink_to_fit(); } if self.cache_size().total() < self.max_cache_size { break; } } From f8dc1f2e3aa96283de3e16d3bbe1e2961b916836 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Mar 2016 12:56:34 +0100 Subject: [PATCH 54/89] Avoid leaking block_details. Fixes #576 --- ethcore/src/blockchain/blockchain.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index eac6da6a2..f412a8240 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -448,7 +448,8 @@ impl BlockChain { let mut write_details = self.block_details.write().unwrap(); for (hash, details) in update.block_details.into_iter() { batch.put_extras(&hash, &details); - write_details.insert(hash, details); + write_details.insert(hash.clone(), details); + self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash)); } let mut write_receipts = self.block_receipts.write().unwrap(); From 8542d651ae8714de4f79342ba9f999dcac8b0988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 4 Mar 2016 11:45:20 +0100 Subject: [PATCH 55/89] Refactoring transactions queue to avoid cloning transactions --- sync/src/transaction_queue.rs | 388 ++++++++++++++++++++-------------- 1 file changed, 224 insertions(+), 164 deletions(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index f2e15f955..1bbfbe36d 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -18,121 +18,126 @@ //! Transaction Queue -use std::vec::Vec; use std::cmp::{Ordering}; use std::collections::{HashMap, BTreeSet}; use util::numbers::{Uint, U256}; -use util::hash::{Address}; +use util::hash::{Address, H256}; use util::table::*; use ethcore::transaction::*; #[derive(Clone, Debug)] -struct VerifiedTransaction { - tx: SignedTransaction, - nonce_height: U256 +struct TransactionOrder { + nonce_height: U256, + gas_price: U256, + hash: H256, } -impl VerifiedTransaction { - pub fn new(tx: SignedTransaction, nonce_height: U256) -> VerifiedTransaction { - VerifiedTransaction { - tx: tx, - nonce_height: nonce_height +impl TransactionOrder { + pub fn for_transaction(tx: &VerifiedTransaction, base_nonce: U256) -> Self { + TransactionOrder { + nonce_height: tx.nonce() - base_nonce, + gas_price: tx.transaction.gas_price, + hash: tx.hash(), } } - - pub fn sender(&self) -> Address { - self.tx.sender().unwrap() - } } -impl Eq for VerifiedTransaction {} -impl PartialEq for VerifiedTransaction { - fn eq(&self, other: &VerifiedTransaction) -> bool { +impl Eq for TransactionOrder {} +impl PartialEq for TransactionOrder { + fn eq(&self, other: &TransactionOrder) -> bool { self.cmp(other) == Ordering::Equal } } -impl PartialOrd for VerifiedTransaction { - fn partial_cmp(&self, other: &VerifiedTransaction) -> Option { +impl PartialOrd for TransactionOrder { + fn partial_cmp(&self, other: &TransactionOrder) -> Option { Some(self.cmp(other)) } } -impl Ord for VerifiedTransaction { - fn cmp(&self, b: &VerifiedTransaction) -> Ordering { +impl Ord for TransactionOrder { + fn cmp(&self, b: &TransactionOrder) -> Ordering { // First check nonce_height if self.nonce_height != b.nonce_height { return self.nonce_height.cmp(&b.nonce_height); } // Then compare gas_prices - let a_gas = self.tx.gas_price; - let b_gas = b.tx.gas_price; + let a_gas = self.gas_price; + let b_gas = b.gas_price; if a_gas != b_gas { return a_gas.cmp(&b_gas); } - // Compare nonce - let a_nonce = self.tx.nonce; - let b_nonce = b.tx.nonce; - if a_nonce != b_nonce { - return a_nonce.cmp(&b_nonce); - } - - // and senders - let a_sender = self.sender(); - let b_sender = b.sender(); - a_sender.cmp(&b_sender) + // Compare hashes + self.hash.cmp(&b.hash) } } -struct TransactionsByPriorityAndAddress { - priority: BTreeSet, - address: Table, +struct VerifiedTransaction { + transaction: SignedTransaction +} +impl VerifiedTransaction { + fn new(transaction: SignedTransaction) -> Self { + VerifiedTransaction { + transaction: transaction + } + } + + fn hash(&self) -> H256 { + self.transaction.hash() + } + + fn nonce(&self) -> U256 { + self.transaction.nonce + } + + fn sender(&self) -> Address { + self.transaction.sender().unwrap() + } +} + +struct TransactionSet { + by_priority: BTreeSet, + by_address: Table, limit: usize, } -impl TransactionsByPriorityAndAddress { - fn insert(&mut self, address: Address, nonce: U256, verified_tx: VerifiedTransaction) { - self.priority.insert(verified_tx.clone()); - self.address.insert(address, nonce, verified_tx); +impl TransactionSet { + fn insert(&mut self, sender: Address, nonce: U256, order: TransactionOrder) { + self.by_priority.insert(order.clone()); + self.by_address.insert(sender, nonce, order); } - fn enforce_limit(&mut self) { - let len = self.priority.len(); + fn enforce_limit(&mut self, by_hash: &HashMap) { + let len = self.by_priority.len(); if len <= self.limit { return; } - let to_remove : Vec = { - self.priority + let to_drop : Vec<&VerifiedTransaction> = { + self.by_priority .iter() .skip(self.limit) - .map(|v_tx| v_tx.tx.clone()) + .map(|order| by_hash.get(&order.hash).expect("Inconsistency in queue detected.")) .collect() }; - for tx in to_remove { - self.remove(&tx); + for tx in to_drop { + self.drop(&tx.sender(), &tx.nonce()); } } - fn remove_by_address(&mut self, sender: &Address, nonce: &U256) -> Option { - if let Some(verified_tx) = self.address.remove(sender, nonce) { - self.priority.remove(&verified_tx); - return Some(verified_tx); + fn drop(&mut self, sender: &Address, nonce: &U256) -> Option { + if let Some(tx_order) = self.by_address.remove(sender, nonce) { + self.by_priority.remove(&tx_order); + return Some(tx_order); } None } - fn remove(&mut self, tx: &SignedTransaction) -> Option { - // First find the transaction by address - let address = tx.sender().unwrap(); - self.remove_by_address(&address, &tx.nonce) - } - fn clear(&mut self) { - self.priority.clear(); - self.address.clear(); + self.by_priority.clear(); + self.by_address.clear(); } } @@ -148,9 +153,11 @@ pub struct TransactionQueueStatus { /// TransactionQueue implementation pub struct TransactionQueue { /// Priority queue for transactions that can go to block - current: TransactionsByPriorityAndAddress, + current: TransactionSet, /// Priority queue for transactions that has been received but are not yet valid to go to block - future: TransactionsByPriorityAndAddress, + future: TransactionSet, + /// All transactions managed by queue indexed by hash + by_hash: HashMap, /// Last nonce of transaction in current last_nonces: HashMap, /// First nonce of transaction in current (used to determine priority) @@ -165,20 +172,21 @@ impl TransactionQueue { /// Create new instance of this Queue with specified limits pub fn with_limits(current_limit: usize, future_limit: usize) -> Self { - let current = TransactionsByPriorityAndAddress { - address: Table::new(), - priority: BTreeSet::new(), + let current = TransactionSet { + by_priority: BTreeSet::new(), + by_address: Table::new(), limit: current_limit, }; - let future = TransactionsByPriorityAndAddress { - address: Table::new(), - priority: BTreeSet::new(), + let future = TransactionSet { + by_priority: BTreeSet::new(), + by_address: Table::new(), limit: future_limit, }; TransactionQueue { current: current, future: future, + by_hash: HashMap::new(), last_nonces: HashMap::new(), first_nonces: HashMap::new(), } @@ -187,8 +195,8 @@ impl TransactionQueue { /// Returns current status for this queue pub fn status(&self) -> TransactionQueueStatus { TransactionQueueStatus { - pending: self.current.priority.len(), - future: self.future.priority.len(), + pending: self.current.by_priority.len(), + future: self.future.by_priority.len(), } } @@ -203,101 +211,107 @@ impl TransactionQueue { /// Add signed transaction to queue to be verified and imported pub fn add(&mut self, tx: SignedTransaction, fetch_nonce: &T) where T: Fn(&Address) -> U256 { - self.import_tx(tx, fetch_nonce); + self.import_tx(VerifiedTransaction::new(tx), fetch_nonce); } - /// Removes all transactions in given slice + /// Removes all transactions identified by hashes given in slice /// /// If gap is introduced marks subsequent transactions as future - pub fn remove_all(&mut self, txs: &[SignedTransaction]) { + pub fn remove_all(&mut self, txs: &[H256]) { for tx in txs { self.remove(&tx); } } - /// Removes transaction from queue. + /// Removes transaction identified by hashes from queue. /// /// If gap is introduced marks subsequent transactions as future - pub fn remove(&mut self, tx: &SignedTransaction) { + pub fn remove(&mut self, hash: &H256) { + let transaction = self.by_hash.remove(hash); + if transaction.is_none() { + // We don't know this transaction + return; + } + let transaction = transaction.unwrap(); + let sender = transaction.sender(); + let nonce = transaction.nonce(); + + // Remove from future + self.future.drop(&sender, &nonce); + // Remove from current - let removed = self.current.remove(tx); - if let Some(verified_tx) = removed { - let sender = verified_tx.sender(); - - // Are there any other transactions from this sender? - if !self.current.address.has_row(&sender) { - // Clear last & first nonces - self.last_nonces.remove(&sender); - self.first_nonces.remove(&sender); - return; - } - - // Let's find those with higher nonce (TODO [todr] optimize?) - let to_move_to_future = { - let row_map = self.current.address.row(&sender).unwrap(); - let tx_nonce = verified_tx.tx.nonce; - let mut to_future = Vec::new(); - let mut highest = U256::zero(); - let mut lowest = tx_nonce.clone(); - - // Search nonces to remove and track lowest and highest - for (nonce, _) in row_map.iter() { - if nonce > &tx_nonce { - to_future.push(nonce.clone()); - } else if nonce > &highest { - highest = nonce.clone(); - } else if nonce < &lowest { - lowest = nonce.clone(); - } - } - - // Update first_nonces and last_nonces - if highest == U256::zero() { - self.last_nonces.remove(&sender); - } else { - self.last_nonces.insert(sender.clone(), highest); - } - - if lowest == tx_nonce { - self.first_nonces.remove(&sender); - } else { - self.first_nonces.insert(sender.clone(), lowest); - } - - // return to future - to_future - }; - - for k in to_move_to_future { - if let Some(v) = self.current.remove_by_address(&sender, &k) { - self.future.insert(sender.clone(), v.tx.nonce, v); - } - } - self.future.enforce_limit(); + let order = self.current.drop(&sender, &nonce); + if order.is_none() { return; } - // Remove from future - { - let sender = tx.sender().unwrap(); - if let Some(_) = self.future.remove_by_address(&sender, &tx.nonce) { - return; + // Are there any other transactions from this sender? + if !self.current.by_address.has_row(&sender) { + // Clear last & first nonces + self.last_nonces.remove(&sender); + self.first_nonces.remove(&sender); + return; + } + + // Let's find those with higher nonce (TODO [todr] optimize?) + let to_move_to_future = { + let row_map = self.current.by_address.row(&sender).unwrap(); + let mut to_future = Vec::new(); + let mut highest = U256::zero(); + let mut lowest = nonce.clone(); + + // Search nonces to remove and track lowest and highest + for (current_nonce, _) in row_map.iter() { + if current_nonce > &nonce { + to_future.push(current_nonce.clone()); + } else if current_nonce > &highest { + highest = current_nonce.clone(); + } else if current_nonce < &lowest { + lowest = current_nonce.clone(); + } + } + + // Update first_nonces and last_nonces + if highest == U256::zero() { + self.last_nonces.remove(&sender); + } else { + self.last_nonces.insert(sender.clone(), highest); + } + + if lowest == nonce { + self.first_nonces.remove(&sender); + } else { + self.first_nonces.insert(sender.clone(), lowest); + } + + // return to future + to_future + }; + + for k in to_move_to_future { + if let Some(v) = self.current.drop(&sender, &k) { + // TODO [todr] Recalculate height? + self.future.insert(sender.clone(), k, v); } } + self.future.enforce_limit(&self.by_hash); } /// Returns top transactions from the queue pub fn top_transactions(&self, size: usize) -> Vec { - self.current.priority + self.current.by_priority .iter() .take(size) - .map(|t| t.tx.clone()).collect() + .map(|t| self.by_hash.get(&t.hash).expect("Transaction Queue Inconsistency")) + .map(|t| t.transaction.clone()) + .collect() } /// Removes all elements (in any state) from the queue pub fn clear(&mut self) { self.current.clear(); self.future.clear(); + self.by_hash.clear(); self.last_nonces.clear(); self.first_nonces.clear(); } @@ -305,31 +319,30 @@ impl TransactionQueue { fn move_future_txs(&mut self, address: Address, current_nonce: U256, first_nonce: U256) -> Option { let mut current_nonce = current_nonce + U256::one(); { - let txs_by_nonce = self.future.address.row_mut(&address); - if let None = txs_by_nonce { + let by_nonce = self.future.by_address.row_mut(&address); + if let None = by_nonce { return None; } - let mut txs_by_nonce = txs_by_nonce.unwrap(); - - while let Some(tx) = txs_by_nonce.remove(¤t_nonce) { - // remove also from priority - self.future.priority.remove(&tx); + let mut by_nonce = by_nonce.unwrap(); + while let Some(order) = by_nonce.remove(¤t_nonce) { + // remove also from priority and hash + self.future.by_priority.remove(&order); // Put to current - let height = current_nonce - first_nonce; - let verified_tx = VerifiedTransaction::new(tx.tx, U256::from(height)); - self.current.insert(address.clone(), verified_tx.tx.nonce, verified_tx); + let transaction = self.by_hash.get(&order.hash).expect("TransactionQueue Inconsistency"); + let order = TransactionOrder::for_transaction(transaction, first_nonce); + self.current.insert(address.clone(), transaction.nonce(), order); current_nonce = current_nonce + U256::one(); } } - self.future.address.clear_if_empty(&address); + self.future.by_address.clear_if_empty(&address); // Returns last inserted nonce Some(current_nonce - U256::one()) } - fn import_tx(&mut self, tx: SignedTransaction, fetch_nonce: &T) + fn import_tx(&mut self, tx: VerifiedTransaction, fetch_nonce: &T) where T: Fn(&Address) -> U256 { - let nonce = tx.nonce; - let address = tx.sender().unwrap(); + let nonce = tx.nonce(); + let address = tx.sender(); let next_nonce = U256::one() + self.last_nonces .get(&address) @@ -338,11 +351,12 @@ impl TransactionQueue { // Check height if nonce > next_nonce { - let height = nonce - next_nonce; - let verified_tx = VerifiedTransaction::new(tx, height); + let order = TransactionOrder::for_transaction(&tx, next_nonce); + // Insert to by_hash + self.by_hash.insert(tx.hash(), tx); // We have a gap - put to future - self.future.insert(address, nonce, verified_tx); - self.future.enforce_limit(); + self.future.insert(address, nonce, order); + self.future.enforce_limit(&self.by_hash); return; } else if next_nonce > nonce { // Droping transaction @@ -354,29 +368,34 @@ impl TransactionQueue { .cloned() .unwrap_or_else(|| nonce.clone()); - let height = nonce - first_nonce; - let verified_tx = VerifiedTransaction::new(tx, height); + let order = TransactionOrder::for_transaction(&tx, first_nonce); + // Insert to by_hash + self.by_hash.insert(tx.hash(), tx); + // Insert to current - self.current.insert(address.clone(), nonce, verified_tx); + self.current.insert(address.clone(), nonce, order); // But maybe there are some more items waiting in future? let new_last_nonce = self.move_future_txs(address.clone(), nonce, first_nonce); self.first_nonces.insert(address.clone(), first_nonce); self.last_nonces.insert(address.clone(), new_last_nonce.unwrap_or(nonce)); // Enforce limit - self.current.enforce_limit(); + self.current.enforce_limit(&self.by_hash); } } + #[cfg(test)] mod test { extern crate rustc_serialize; use self::rustc_serialize::hex::FromHex; - + use std::collections::{HashMap, BTreeSet}; use util::crypto::KeyPair; use util::numbers::{U256, Uint}; use util::hash::{Address}; + use util::table::*; use ethcore::transaction::*; use super::*; + use super::{TransactionSet, TransactionOrder, VerifiedTransaction}; fn new_unsigned_tx(nonce: U256) -> Transaction { Transaction { @@ -408,6 +427,46 @@ mod test { (tx.sign(secret), tx2.sign(secret)) } + #[test] + fn should_create_transaction_set() { + // given + let mut set = TransactionSet { + by_priority: BTreeSet::new(), + by_address: Table::new(), + limit: 1 + }; + let (tx1, tx2) = new_txs(U256::from(1)); + let tx1 = VerifiedTransaction::new(tx1); + let tx2 = VerifiedTransaction::new(tx2); + let by_hash = { + let mut x = HashMap::new(); + let tx1 = VerifiedTransaction::new(tx1.transaction.clone()); + let tx2 = VerifiedTransaction::new(tx2.transaction.clone()); + x.insert(tx1.hash(), tx1); + x.insert(tx2.hash(), tx2); + x + }; + // Insert both transactions + let order1 = TransactionOrder::for_transaction(&tx1, U256::zero()); + set.insert(tx1.sender(), tx1.nonce(), order1.clone()); + let order2 = TransactionOrder::for_transaction(&tx2, U256::zero()); + set.insert(tx2.sender(), tx2.nonce(), order2.clone()); + assert_eq!(set.by_priority.len(), 2); + assert_eq!(set.by_address.len(), 2); + + // when + set.enforce_limit(&by_hash); + + // then + assert_eq!(set.by_priority.len(), 1); + assert_eq!(set.by_address.len(), 1); + assert_eq!(set.by_priority.iter().next().unwrap().clone(), order1); + set.clear(); + assert_eq!(set.by_priority.len(), 0); + assert_eq!(set.by_address.len(), 0); + } + + #[test] fn should_import_tx() { // given @@ -495,8 +554,8 @@ mod test { assert_eq!(txq2.status().future, 1); // when - txq2.remove(&tx); - txq2.remove(&tx2); + txq2.remove(&tx.hash()); + txq2.remove(&tx2.hash()); // then @@ -518,7 +577,7 @@ mod test { assert_eq!(txq.status().pending, 3); // when - txq.remove(&tx); + txq.remove(&tx.hash()); // then let stats = txq.status(); @@ -608,14 +667,15 @@ mod test { assert_eq!(txq.status().pending, 2); // when - txq.remove(&tx1); + txq.remove(&tx1.hash()); + assert_eq!(txq.status().pending, 0); assert_eq!(txq.status().future, 1); txq.add(tx1.clone(), &default_nonce); // then let stats = txq.status(); - assert_eq!(stats.pending, 2); assert_eq!(stats.future, 0); + assert_eq!(stats.pending, 2); } From 706ce5dfb62bb0cc5f6c638b037c28c6f78a1e1d Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Mar 2016 11:56:04 +0100 Subject: [PATCH 56/89] verifier trait --- ethcore/src/client.rs | 18 +++++++++--- ethcore/src/verification/canon_verifier.rs | 28 +++++++++++++++++++ ethcore/src/verification/mod.rs | 25 +++++++++++++++++ ethcore/src/verification/noop_verifier.rs | 27 ++++++++++++++++++ .../src/{ => verification}/verification.rs | 6 ++-- ethcore/src/verification/verifier.rs | 23 +++++++++++++++ 6 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 ethcore/src/verification/canon_verifier.rs create mode 100644 ethcore/src/verification/mod.rs create mode 100644 ethcore/src/verification/noop_verifier.rs rename ethcore/src/{ => verification}/verification.rs (99%) create mode 100644 ethcore/src/verification/verifier.rs diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 422f1eaa2..6ae628f8b 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -16,6 +16,7 @@ //! Blockchain database client. +use std::marker::PhantomData; use util::*; use util::panics::*; use blockchain::{BlockChain, BlockProvider}; @@ -188,7 +189,7 @@ impl ClientReport { /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. /// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue. -pub struct Client { +pub struct Client where V: Verifier { chain: Arc>, engine: Arc>, state_db: Mutex, @@ -201,14 +202,22 @@ pub struct Client { sealing_block: Mutex>, author: RwLock
, extra_data: RwLock, + verifier: PhantomData } const HISTORY: u64 = 1000; const CLIENT_DB_VER_STR: &'static str = "4.0"; -impl Client { +impl Client { /// Create a new client with given spec and DB path. pub fn new(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + Client::::new_with_verifier(config, spec, path, message_channel) + } +} + +impl Client where V: Verifier { + /// Create a new client with given spec and DB path and custom verifier. + pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning @@ -240,6 +249,7 @@ impl Client { sealing_block: Mutex::new(None), author: RwLock::new(Address::new()), extra_data: RwLock::new(Vec::new()), + verifier: PhantomData })) } @@ -302,7 +312,7 @@ impl Client { // Final Verification let closed_block = enact_result.unwrap(); - if let Err(e) = verify_block_final(&header, closed_block.block().header()) { + if let Err(e) = V::verify_block_final(&header, closed_block.block().header()) { warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); } @@ -503,7 +513,7 @@ impl Client { // TODO: need MinerService MinerIoHandler -impl BlockChainClient for Client { +impl BlockChainClient for Client where V: Verifier { fn block_header(&self, id: BlockId) -> Option { let chain = self.chain.read().unwrap(); Self::block_hash(&chain, id).and_then(|hash| chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec())) diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs new file mode 100644 index 000000000..0d9cbc6b6 --- /dev/null +++ b/ethcore/src/verification/canon_verifier.rs @@ -0,0 +1,28 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use error::Error; +use header::Header; +use super::Verifier; +use super::verification; + +pub struct CanonVerifier; + +impl Verifier for CanonVerifier { + fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> { + verification::verify_block_final(expected, got) + } +} diff --git a/ethcore/src/verification/mod.rs b/ethcore/src/verification/mod.rs new file mode 100644 index 000000000..260121989 --- /dev/null +++ b/ethcore/src/verification/mod.rs @@ -0,0 +1,25 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +pub mod verification; +pub mod verifier; +mod canon_verifier; +mod noop_verifier; + +pub use self::verification::*; +pub use self::verifier::Verifier; +pub use self::canon_verifier::CanonVerifier; +pub use self::noop_verifier::NoopVerifier; diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs new file mode 100644 index 000000000..8dfd64771 --- /dev/null +++ b/ethcore/src/verification/noop_verifier.rs @@ -0,0 +1,27 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use error::Error; +use header::Header; +use super::Verifier; + +pub struct NoopVerifier; + +impl Verifier for NoopVerifier { + fn verify_block_final(_expected: &Header, _got: &Header) -> Result<(), Error> { + Ok(()) + } +} diff --git a/ethcore/src/verification.rs b/ethcore/src/verification/verification.rs similarity index 99% rename from ethcore/src/verification.rs rename to ethcore/src/verification/verification.rs index f52e2e1e4..fa540e7c6 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -61,7 +61,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { try!(engine.verify_block_unordered(&u, None)); } - // Verify transactions. + // Verify transactions. let mut transactions = Vec::new(); { let v = BlockView::new(&bytes); @@ -141,7 +141,7 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, b let uncle_parent = try!(bc.block_header(&uncle.parent_hash).ok_or_else(|| Error::from(BlockError::UnknownUncleParent(uncle.parent_hash.clone())))); for _ in 0..depth { match bc.block_details(&expected_uncle_parent) { - Some(details) => { + Some(details) => { expected_uncle_parent = details.parent; }, None => break @@ -468,7 +468,7 @@ mod tests { header.number = 9; check_fail(family_test(&create_test_block_with_data(&header, &good_transactions, &good_uncles), engine.deref(), &bc), InvalidNumber(Mismatch { expected: parent.number + 1, found: header.number })); - + header = good.clone(); let mut bad_uncles = good_uncles.clone(); bad_uncles.push(good_uncle1.clone()); diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs new file mode 100644 index 000000000..0ffbf3bdd --- /dev/null +++ b/ethcore/src/verification/verifier.rs @@ -0,0 +1,23 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use error::Error; +use header::Header; + +/// Should be used to verify blocks. +pub trait Verifier: Send + Sync { + fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>; +} From 6a57e83509f9f468e477950270b8318d1cec06b0 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 14:46:54 +0300 Subject: [PATCH 57/89] json rpc personal service --- rpc/src/v1/impls/mod.rs | 2 + rpc/src/v1/impls/personal.rs | 81 +++++++++++++++++++++++++++++++++++ rpc/src/v1/mod.rs | 4 +- rpc/src/v1/traits/mod.rs | 2 + rpc/src/v1/traits/personal.rs | 41 ++++++++++++++++++ 5 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 rpc/src/v1/impls/personal.rs create mode 100644 rpc/src/v1/traits/personal.rs diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs index d9102b1db..10d451e9f 100644 --- a/rpc/src/v1/impls/mod.rs +++ b/rpc/src/v1/impls/mod.rs @@ -28,7 +28,9 @@ macro_rules! take_weak { mod web3; mod eth; mod net; +mod personal; pub use self::web3::Web3Client; pub use self::eth::{EthClient, EthFilterClient}; pub use self::net::NetClient; +pub use self::personal::PersonalClient; diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs new file mode 100644 index 000000000..b97449541 --- /dev/null +++ b/rpc/src/v1/impls/personal.rs @@ -0,0 +1,81 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Net rpc implementation. +use std::sync::{Arc, Weak}; +use jsonrpc_core::*; +use v1::traits::Personal; +use util::keys::store::*; +use util::{Bytes, Address}; +use std::sync::RwLock; + +/// Net rpc implementation. +pub struct PersonalClient { + secret_store: Weak, + unlocked_account: Arc>>, + unlocked_secret: Arc>>, +} + +impl PersonalClient { + /// Creates new PersonalClient + pub fn new(store: &Arc) -> Self { + PersonalClient { + secret_store: Arc::downgrade(store), + unlocked_account: Arc::new(RwLock::new(None)), + unlocked_secret: Arc::new(RwLock::new(None)), + } + } +} + +impl Personal for PersonalClient { + fn accounts(&self, _: Params) -> Result { + let store = take_weak!(self.secret_store); + match store.accounts() { + Ok(account_list) => { + Ok(Value::Array(account_list.iter() + .map(|&(account, _)| Value::String(format!("{:?}", account))) + .collect::>()) + ) + } + Err(_) => Err(Error::internal_error()) + } + } + + fn new_account(&self, _: Params) -> Result { + Err(Error::internal_error()) + } + + fn unlock_account(&self, params: Params) -> Result { + from_params::<(Address, String, u64)>(params).and_then( + |(account, account_pass, _)|{ + let store = take_weak!(self.secret_store); + let secret_id = match store.account(&account) { + None => { return Ok(Value::Bool(false)); } + Some(id) => id + }; + match store.get(&secret_id, &account_pass) { + Ok(secret) => { + *self.unlocked_account.write().unwrap() = Some(account); + *self.unlocked_secret.write().unwrap() = Some(secret); + Ok(Value::Bool(true)) + }, + Err(_) => { + Ok(Value::Bool(false)) + } + } + }) + } +} diff --git a/rpc/src/v1/mod.rs b/rpc/src/v1/mod.rs index 57f7a944a..104a8b3f0 100644 --- a/rpc/src/v1/mod.rs +++ b/rpc/src/v1/mod.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Ethcore rpc v1. -//! +//! //! Compliant with ethereum rpc. pub mod traits; @@ -25,5 +25,5 @@ mod types; mod tests; mod helpers; -pub use self::traits::{Web3, Eth, EthFilter, Net}; +pub use self::traits::{Web3, Eth, EthFilter, Personal, Net}; pub use self::impls::*; diff --git a/rpc/src/v1/traits/mod.rs b/rpc/src/v1/traits/mod.rs index c9af6dac3..0a95cb050 100644 --- a/rpc/src/v1/traits/mod.rs +++ b/rpc/src/v1/traits/mod.rs @@ -23,7 +23,9 @@ macro_rules! rpc_unimplemented { pub mod web3; pub mod eth; pub mod net; +pub mod personal; pub use self::web3::Web3; pub use self::eth::{Eth, EthFilter}; pub use self::net::Net; +pub use self::personal::Personal; diff --git a/rpc/src/v1/traits/personal.rs b/rpc/src/v1/traits/personal.rs new file mode 100644 index 000000000..0cf72951c --- /dev/null +++ b/rpc/src/v1/traits/personal.rs @@ -0,0 +1,41 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Personal rpc interface. +use std::sync::Arc; +use jsonrpc_core::*; + +/// Personal rpc interface. +pub trait Personal: Sized + Send + Sync + 'static { + + /// Lists all stored accounts + fn accounts(&self, _: Params) -> Result { rpc_unimplemented!() } + + /// Creates new account (it becomes new current unlocked account) + fn new_account(&self, _: Params) -> Result { rpc_unimplemented!() } + + /// Unlocks specified account for use (can only be one unlocked account at one moment) + fn unlock_account(&self, _: Params) -> Result { rpc_unimplemented!() } + + /// Should be used to convert object to io delegate. + fn to_delegate(self) -> IoDelegate { + let mut delegate = IoDelegate::new(Arc::new(self)); + delegate.add_method("personal_listAccounts", Personal::accounts); + delegate.add_method("personal_newAccount", Personal::new_account); + delegate.add_method("personal_unlockAccount", Personal::unlock_account); + delegate + } +} From e17b2a4db8871e9e70917a75e7975d32d652ecb4 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 14:48:05 +0300 Subject: [PATCH 58/89] replacing unsafe cell with rwlock --- util/src/keys/directory.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 7178508a2..d0d3393cd 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -461,17 +461,17 @@ enum KeyFileLoadError { pub struct KeyDirectory { /// Directory path for key management. path: String, - cache: RefCell>, - cache_usage: RefCell>, + cache: RwLock>, + cache_usage: RwLock>, } impl KeyDirectory { /// Initializes new cache directory context with a given `path` pub fn new(path: &Path) -> KeyDirectory { KeyDirectory { - cache: RefCell::new(HashMap::new()), + cache: RwLock::new(HashMap::new()), path: path.to_str().expect("Initialized key directory with empty path").to_owned(), - cache_usage: RefCell::new(VecDeque::new()), + cache_usage: RwLock::new(VecDeque::new()), } } @@ -484,7 +484,7 @@ impl KeyDirectory { let json_bytes = json_text.into_bytes(); try!(file.write(&json_bytes)); } - let mut cache = self.cache.borrow_mut(); + let mut cache = self.cache.write().unwrap(); let id = key_file.id.clone(); cache.insert(id.clone(), key_file); Ok(id.clone()) @@ -495,14 +495,14 @@ impl KeyDirectory { pub fn get(&self, id: &Uuid) -> Option { let path = self.key_path(id); { - let mut usage = self.cache_usage.borrow_mut(); + let mut usage = self.cache_usage.write().unwrap(); usage.push_back(id.clone()); } - if !self.cache.borrow().contains_key(id) { + if !self.cache.read().unwrap().contains_key(id) { match KeyDirectory::load_key(&path) { Ok(loaded_key) => { - self.cache.borrow_mut().insert(id.to_owned(), loaded_key); + self.cache.write().unwrap().insert(id.to_owned(), loaded_key); } Err(error) => { warn!(target: "sstore", "error loading key {:?}: {:?}", id, error); @@ -512,7 +512,7 @@ impl KeyDirectory { } // todo: replace with Ref::map when it stabilized to avoid copies - Some(self.cache.borrow().get(id) + Some(self.cache.read().unwrap().get(id) .expect("Key should be there, we have just inserted or checked it.") .clone()) } @@ -524,7 +524,7 @@ impl KeyDirectory { /// Removes keys that never been requested during last `MAX_USAGE_TRACK` times pub fn collect_garbage(&mut self) { - let mut cache_usage = self.cache_usage.borrow_mut(); + let mut cache_usage = self.cache_usage.write().unwrap(); let total_usages = cache_usage.len(); let untracked_usages = max(total_usages as i64 - MAX_CACHE_USAGE_TRACK as i64, 0) as usize; @@ -532,31 +532,31 @@ impl KeyDirectory { cache_usage.drain(..untracked_usages); } - if self.cache.borrow().len() <= MAX_CACHE_USAGE_TRACK { return; } + if self.cache.read().unwrap().len() <= MAX_CACHE_USAGE_TRACK { return; } let uniqs: HashSet<&Uuid> = cache_usage.iter().collect(); let removes:Vec = { - let cache = self.cache.borrow(); + let cache = self.cache.read().unwrap(); cache.keys().cloned().filter(|key| !uniqs.contains(key)).collect() }; if removes.is_empty() { return; } - let mut cache = self.cache.borrow_mut(); + let mut cache = self.cache.write().unwrap(); for key in removes { cache.remove(&key); } } /// Reports how many keys are currently cached. pub fn cache_size(&self) -> usize { - self.cache.borrow().len() + self.cache.read().unwrap().len() } /// Removes key file from key directory pub fn delete(&mut self, id: &Uuid) -> Result<(), ::std::io::Error> { let path = self.key_path(id); - if !self.cache.borrow().contains_key(id) { + if !self.cache.read().unwrap().contains_key(id) { return match fs::remove_file(&path) { Ok(_) => { - self.cache.borrow_mut().remove(&id); + self.cache.write().unwrap().remove(&id); Ok(()) }, Err(e) => Err(e) From 4f62e80de753d40e821540034771123c5da895e0 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 14:53:18 +0300 Subject: [PATCH 59/89] name fix --- rpc/src/v1/impls/personal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index b97449541..d7066c082 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Net rpc implementation. +//! Account management (personal) rpc implementation use std::sync::{Arc, Weak}; use jsonrpc_core::*; use v1::traits::Personal; @@ -22,7 +22,7 @@ use util::keys::store::*; use util::{Bytes, Address}; use std::sync::RwLock; -/// Net rpc implementation. +/// Account management (personal) rpc implementation. pub struct PersonalClient { secret_store: Weak, unlocked_account: Arc>>, From 0d01099f44374d3bc10515b0a215d3eed08dd9c4 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 16:23:00 +0300 Subject: [PATCH 60/89] moving unlock logics to secret-store itself --- Cargo.lock | 10 ++++++ rpc/src/v1/impls/personal.rs | 45 +++++++++++++-------------- util/Cargo.toml | 1 + util/src/keys/store.rs | 60 ++++++++++++++++++++++++++++++++++-- util/src/lib.rs | 1 + 5 files changed, 90 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03b8092a0..17f2d87d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,15 @@ name = "cfg-if" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "chrono" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clippy" version = "0.0.44" @@ -236,6 +245,7 @@ version = "0.9.99" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "bigint 0.1.0", + "chrono 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)", "clippy 0.0.44 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index d7066c082..48e1b1c6a 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -19,30 +19,27 @@ use std::sync::{Arc, Weak}; use jsonrpc_core::*; use v1::traits::Personal; use util::keys::store::*; -use util::{Bytes, Address}; +use util::Address; use std::sync::RwLock; /// Account management (personal) rpc implementation. pub struct PersonalClient { - secret_store: Weak, - unlocked_account: Arc>>, - unlocked_secret: Arc>>, + secret_store: Weak>, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(store: &Arc) -> Self { + pub fn new(store: &Arc>) -> Self { PersonalClient { secret_store: Arc::downgrade(store), - unlocked_account: Arc::new(RwLock::new(None)), - unlocked_secret: Arc::new(RwLock::new(None)), } } } impl Personal for PersonalClient { fn accounts(&self, _: Params) -> Result { - let store = take_weak!(self.secret_store); + let store_wk = take_weak!(self.secret_store); + let store = store_wk.read().unwrap(); match store.accounts() { Ok(account_list) => { Ok(Value::Array(account_list.iter() @@ -54,27 +51,27 @@ impl Personal for PersonalClient { } } - fn new_account(&self, _: Params) -> Result { - Err(Error::internal_error()) + fn new_account(&self, params: Params) -> Result { + from_params::<(String, )>(params).and_then( + |(pass, )| { + let store_wk = take_weak!(self.secret_store); + let mut store = store_wk.write().unwrap(); + match store.new_account(&pass) { + Ok(address) => Ok(Value::String(format!("{:?}", address))), + Err(_) => Err(Error::internal_error()) + } + } + ) } fn unlock_account(&self, params: Params) -> Result { from_params::<(Address, String, u64)>(params).and_then( |(account, account_pass, _)|{ - let store = take_weak!(self.secret_store); - let secret_id = match store.account(&account) { - None => { return Ok(Value::Bool(false)); } - Some(id) => id - }; - match store.get(&secret_id, &account_pass) { - Ok(secret) => { - *self.unlocked_account.write().unwrap() = Some(account); - *self.unlocked_secret.write().unwrap() = Some(secret); - Ok(Value::Bool(true)) - }, - Err(_) => { - Ok(Value::Bool(false)) - } + let store_wk = take_weak!(self.secret_store); + let store = store_wk.read().unwrap(); + match store.unlock_account(&account, &account_pass) { + Ok(_) => Ok(Value::Bool(true)), + Err(_) => Ok(Value::Bool(false)), } }) } diff --git a/util/Cargo.toml b/util/Cargo.toml index 0c7df3f40..9c5cb3fe3 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -36,6 +36,7 @@ libc = "0.2.7" vergen = "0.1" target_info = "0.1" bigint = { path = "bigint" } +chrono = "0.2" [features] default = [] diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index c4fa377f9..99cbf751b 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -22,6 +22,7 @@ use rcrypto::pbkdf2::*; use rcrypto::scrypt::*; use rcrypto::hmac::*; use crypto; +use chrono::*; const KEY_LENGTH: u32 = 32; const KEY_ITERATIONS: u32 = 10240; @@ -57,7 +58,13 @@ pub enum EncryptedHashMapError { /// Represent service for storing encrypted arbitrary data pub struct SecretStore { - directory: KeyDirectory + directory: KeyDirectory, + unlocks: RwLock> +} + +struct AccountUnlock { + secret: H256, + expires: DateTime, } impl SecretStore { @@ -72,7 +79,8 @@ impl SecretStore { /// new instance of Secret Store in specific directory pub fn new_in(path: &Path) -> SecretStore { SecretStore { - directory: KeyDirectory::new(path) + directory: KeyDirectory::new(path), + unlocks: RwLock::new(HashMap::new()) } } @@ -120,9 +128,37 @@ impl SecretStore { #[cfg(test)] fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { - directory: KeyDirectory::new(path.as_path()) + directory: KeyDirectory::new(path.as_path()), + unlocks: RwLock::new(HashMap::new()) } } + + /// Unlocks account for use + pub fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { + let secret_id = try!(self.account(&account).ok_or(EncryptedHashMapError::UnknownIdentifier)); + let secret = try!(self.get(&secret_id, pass)); + { + let mut write_lock = self.unlocks.write().unwrap(); + let mut unlock = write_lock.entry(*account) + .or_insert_with(|| AccountUnlock { secret: secret, expires: UTC::now() }); + unlock.secret = secret; + unlock.expires = UTC::now() + Duration::minutes(20); + } + Ok(()) + } + + /// Creates new account + pub fn new_account(&mut self, pass: &str) -> Result { + let secret = H256::random(); + let key_id = H128::random(); + self.insert(key_id.clone(), secret, pass); + + let mut key_file = self.directory.get(&key_id).expect("the key was just inserted"); + let address = Address::random(); + key_file.account = Some(address); + try!(self.directory.save(key_file)); + Ok(address) + } } fn derive_key_iterations(password: &str, salt: &H256, c: u32) -> (Bytes, Bytes) { @@ -369,6 +405,24 @@ mod tests { assert_eq!(4, sstore.directory.list().unwrap().len()) } + #[test] + fn can_create_account() { + let temp = RandomTempPath::create_dir(); + let mut sstore = SecretStore::new_test(&temp); + sstore.new_account("123").unwrap(); + assert_eq!(1, sstore.accounts().unwrap().len()); + } + + #[test] + fn can_unlock_account() { + let temp = RandomTempPath::create_dir(); + let mut sstore = SecretStore::new_test(&temp); + let address = sstore.new_account("123").unwrap(); + + let secret = sstore.unlock_account(&address, "123"); + assert!(secret.is_ok()); + } + #[test] fn can_import_account() { use keys::directory::{KeyFileContent, KeyFileCrypto}; diff --git a/util/src/lib.rs b/util/src/lib.rs index 8594e6f40..a50ba8da4 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -111,6 +111,7 @@ extern crate rustc_version; extern crate target_info; extern crate vergen; extern crate bigint; +extern crate chrono; pub mod standard; #[macro_use] From ae51d99fb89e3311196a5d2f491812845169d834 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 16:40:17 +0300 Subject: [PATCH 61/89] [ci skip] trailing commas --- util/src/keys/store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 99cbf751b..f05b8dec4 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -59,7 +59,7 @@ pub enum EncryptedHashMapError { /// Represent service for storing encrypted arbitrary data pub struct SecretStore { directory: KeyDirectory, - unlocks: RwLock> + unlocks: RwLock>, } struct AccountUnlock { @@ -80,7 +80,7 @@ impl SecretStore { pub fn new_in(path: &Path) -> SecretStore { SecretStore { directory: KeyDirectory::new(path), - unlocks: RwLock::new(HashMap::new()) + unlocks: RwLock::new(HashMap::new()), } } @@ -129,7 +129,7 @@ impl SecretStore { fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { directory: KeyDirectory::new(path.as_path()), - unlocks: RwLock::new(HashMap::new()) + unlocks: RwLock::new(HashMap::new()), } } From b320ff46020ff8a3f517901969d17a28fd040de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 4 Mar 2016 15:02:11 +0100 Subject: [PATCH 62/89] Getting rid of first_nonces (we can fetch it from state) --- sync/src/transaction_queue.rs | 76 ++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 1bbfbe36d..a12d75ed4 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -158,10 +158,8 @@ pub struct TransactionQueue { future: TransactionSet, /// All transactions managed by queue indexed by hash by_hash: HashMap, - /// Last nonce of transaction in current + /// Last nonce of transaction in current (to quickly check next expected transaction) last_nonces: HashMap, - /// First nonce of transaction in current (used to determine priority) - first_nonces: HashMap, } impl TransactionQueue { @@ -188,7 +186,6 @@ impl TransactionQueue { future: future, by_hash: HashMap::new(), last_nonces: HashMap::new(), - first_nonces: HashMap::new(), } } @@ -217,16 +214,18 @@ impl TransactionQueue { /// Removes all transactions identified by hashes given in slice /// /// If gap is introduced marks subsequent transactions as future - pub fn remove_all(&mut self, txs: &[H256]) { + pub fn remove_all(&mut self, txs: &[H256], fetch_nonce: T) + where T: Fn(&Address) -> U256 { for tx in txs { - self.remove(&tx); + self.remove(&tx, &fetch_nonce); } } /// Removes transaction identified by hashes from queue. /// /// If gap is introduced marks subsequent transactions as future - pub fn remove(&mut self, hash: &H256) { + pub fn remove(&mut self, hash: &H256, fetch_nonce: &T) + where T: Fn(&Address) -> U256 { let transaction = self.by_hash.remove(hash); if transaction.is_none() { // We don't know this transaction @@ -249,11 +248,10 @@ impl TransactionQueue { if !self.current.by_address.has_row(&sender) { // Clear last & first nonces self.last_nonces.remove(&sender); - self.first_nonces.remove(&sender); return; } - // Let's find those with higher nonce (TODO [todr] optimize?) + // Let's find those with higher nonce that should be moved to future (TODO [todr] optimize?) let to_move_to_future = { let row_map = self.current.by_address.row(&sender).unwrap(); let mut to_future = Vec::new(); @@ -271,23 +269,17 @@ impl TransactionQueue { } } - // Update first_nonces and last_nonces + // Update last_nonces if highest == U256::zero() { self.last_nonces.remove(&sender); } else { self.last_nonces.insert(sender.clone(), highest); } - if lowest == nonce { - self.first_nonces.remove(&sender); - } else { - self.first_nonces.insert(sender.clone(), lowest); - } - - // return to future to_future }; + // Move to future for k in to_move_to_future { if let Some(v) = self.current.drop(&sender, &k) { // TODO [todr] Recalculate height? @@ -295,6 +287,8 @@ impl TransactionQueue { } } self.future.enforce_limit(&self.by_hash); + + // But maybe some transactions } /// Returns top transactions from the queue @@ -313,7 +307,6 @@ impl TransactionQueue { self.future.clear(); self.by_hash.clear(); self.last_nonces.clear(); - self.first_nonces.clear(); } fn move_future_txs(&mut self, address: Address, current_nonce: U256, first_nonce: U256) -> Option { @@ -344,9 +337,10 @@ impl TransactionQueue { let nonce = tx.nonce(); let address = tx.sender(); - let next_nonce = U256::one() + self.last_nonces + let next_nonce = self.last_nonces .get(&address) .cloned() + .map(|n| n + U256::one()) .unwrap_or_else(|| fetch_nonce(&address)); // Check height @@ -363,20 +357,15 @@ impl TransactionQueue { return; } - let first_nonce = self.first_nonces - .get(&address) - .cloned() - .unwrap_or_else(|| nonce.clone()); - - let order = TransactionOrder::for_transaction(&tx, first_nonce); + let base_nonce = fetch_nonce(&address); + let order = TransactionOrder::for_transaction(&tx, base_nonce); // Insert to by_hash self.by_hash.insert(tx.hash(), tx); // Insert to current self.current.insert(address.clone(), nonce, order); // But maybe there are some more items waiting in future? - let new_last_nonce = self.move_future_txs(address.clone(), nonce, first_nonce); - self.first_nonces.insert(address.clone(), first_nonce); + let new_last_nonce = self.move_future_txs(address.clone(), nonce, base_nonce); self.last_nonces.insert(address.clone(), new_last_nonce.unwrap_or(nonce)); // Enforce limit self.current.enforce_limit(&self.by_hash); @@ -414,7 +403,7 @@ mod test { } fn default_nonce(_address: &Address) -> U256 { - U256::from(122) + U256::from(123) } fn new_txs(second_nonce: U256) -> (SignedTransaction, SignedTransaction) { @@ -554,8 +543,8 @@ mod test { assert_eq!(txq2.status().future, 1); // when - txq2.remove(&tx.hash()); - txq2.remove(&tx2.hash()); + txq2.remove(&tx.hash(), &default_nonce); + txq2.remove(&tx2.hash(), &default_nonce); // then @@ -577,7 +566,7 @@ mod test { assert_eq!(txq.status().pending, 3); // when - txq.remove(&tx.hash()); + txq.remove(&tx.hash(), &default_nonce); // then let stats = txq.status(); @@ -645,7 +634,7 @@ mod test { fn should_drop_transactions_with_old_nonces() { let mut txq = TransactionQueue::new(); let tx = new_tx(); - let last_nonce = tx.nonce.clone(); + let last_nonce = tx.nonce.clone() + U256::one(); let fetch_last_nonce = |_a: &Address| last_nonce; // when @@ -667,7 +656,7 @@ mod test { assert_eq!(txq.status().pending, 2); // when - txq.remove(&tx1.hash()); + txq.remove(&tx1.hash(), &default_nonce); assert_eq!(txq.status().pending, 0); assert_eq!(txq.status().future, 1); txq.add(tx1.clone(), &default_nonce); @@ -676,7 +665,28 @@ mod test { let stats = txq.status(); assert_eq!(stats.future, 0); assert_eq!(stats.pending, 2); + } + #[test] + fn should_not_move_to_future_if_state_nonce_is_higher() { + // given + let next_nonce = |a: &Address| default_nonce(a) + U256::one(); + let mut txq = TransactionQueue::new(); + let (tx, tx2) = new_txs(U256::from(1)); + let tx3 = new_tx(); + txq.add(tx2.clone(), &default_nonce); + assert_eq!(txq.status().future, 1); + txq.add(tx3.clone(), &default_nonce); + txq.add(tx.clone(), &default_nonce); + assert_eq!(txq.status().pending, 3); + + // when + txq.remove(&tx.hash(), &next_nonce); + + // then + let stats = txq.status(); + assert_eq!(stats.future, 0); + assert_eq!(stats.pending, 2); } } From 677c3996b9aca31debd3059b1e3e575dce99ffb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 4 Mar 2016 16:09:05 +0100 Subject: [PATCH 63/89] Taking expected nonce from state into consideration when removing txs --- sync/src/transaction_queue.rs | 79 ++++++++++++++++------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index a12d75ed4..4f5622a2f 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -34,13 +34,18 @@ struct TransactionOrder { } impl TransactionOrder { - pub fn for_transaction(tx: &VerifiedTransaction, base_nonce: U256) -> Self { + fn for_transaction(tx: &VerifiedTransaction, base_nonce: U256) -> Self { TransactionOrder { nonce_height: tx.nonce() - base_nonce, gas_price: tx.transaction.gas_price, hash: tx.hash(), } } + + fn update_height(mut self, nonce: U256, base_nonce: U256) -> Self { + self.nonce_height = nonce - base_nonce; + self + } } impl Eq for TransactionOrder {} @@ -235,6 +240,7 @@ impl TransactionQueue { let sender = transaction.sender(); let nonce = transaction.nonce(); + println!("Removing tx: {:?}", transaction.transaction); // Remove from future self.future.drop(&sender, &nonce); @@ -244,51 +250,35 @@ impl TransactionQueue { return; } - // Are there any other transactions from this sender? - if !self.current.by_address.has_row(&sender) { - // Clear last & first nonces - self.last_nonces.remove(&sender); - return; - } + // Let's remove transactions where tx.nonce < current_nonce + // and if there are any future transactions matching current_nonce+1 - move to current + let current_nonce = fetch_nonce(&sender); + // We will either move transaction to future or remove it completely + // so there will be no transactions from this sender in current + self.last_nonces.remove(&sender); - // Let's find those with higher nonce that should be moved to future (TODO [todr] optimize?) - let to_move_to_future = { - let row_map = self.current.by_address.row(&sender).unwrap(); - let mut to_future = Vec::new(); - let mut highest = U256::zero(); - let mut lowest = nonce.clone(); - - // Search nonces to remove and track lowest and highest - for (current_nonce, _) in row_map.iter() { - if current_nonce > &nonce { - to_future.push(current_nonce.clone()); - } else if current_nonce > &highest { - highest = current_nonce.clone(); - } else if current_nonce < &lowest { - lowest = current_nonce.clone(); - } - } - - // Update last_nonces - if highest == U256::zero() { - self.last_nonces.remove(&sender); - } else { - self.last_nonces.insert(sender.clone(), highest); - } - - to_future + let all_nonces_from_sender = match self.current.by_address.row(&sender) { + Some(row_map) => row_map.keys().cloned().collect::>(), + None => vec![], }; - // Move to future - for k in to_move_to_future { - if let Some(v) = self.current.drop(&sender, &k) { - // TODO [todr] Recalculate height? - self.future.insert(sender.clone(), k, v); + for k in all_nonces_from_sender { + // Goes to future or is removed + let order = self.current.drop(&sender, &k).unwrap(); + if k >= current_nonce { + println!("Moving to future: {:?}", order); + self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); + } else { + self.by_hash.remove(&order.hash); } } self.future.enforce_limit(&self.by_hash); - // But maybe some transactions + // And now lets check if there is some chain of transactions in future + // that should be placed in current + if let Some(new_current_top) = self.move_future_txs(sender.clone(), current_nonce - U256::one(), current_nonce) { + self.last_nonces.insert(sender, new_current_top); + } } /// Returns top transactions from the queue @@ -310,6 +300,7 @@ impl TransactionQueue { } fn move_future_txs(&mut self, address: Address, current_nonce: U256, first_nonce: U256) -> Option { + println!("Moving from future for: {:?} base: {:?}", current_nonce, first_nonce); let mut current_nonce = current_nonce + U256::one(); { let by_nonce = self.future.by_address.row_mut(&address); @@ -321,9 +312,9 @@ impl TransactionQueue { // remove also from priority and hash self.future.by_priority.remove(&order); // Put to current - let transaction = self.by_hash.get(&order.hash).expect("TransactionQueue Inconsistency"); - let order = TransactionOrder::for_transaction(transaction, first_nonce); - self.current.insert(address.clone(), transaction.nonce(), order); + println!("Moved: {:?}", order); + let order = order.update_height(current_nonce.clone(), first_nonce); + self.current.insert(address.clone(), current_nonce, order); current_nonce = current_nonce + U256::one(); } } @@ -340,9 +331,9 @@ impl TransactionQueue { let next_nonce = self.last_nonces .get(&address) .cloned() - .map(|n| n + U256::one()) - .unwrap_or_else(|| fetch_nonce(&address)); + .map_or_else(|| fetch_nonce(&address), |n| n + U256::one()); + println!("Expected next: {:?}, got: {:?}", next_nonce, nonce); // Check height if nonce > next_nonce { let order = TransactionOrder::for_transaction(&tx, next_nonce); From bcaed67eaa0257c8998925287fd2dbc40df12698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 4 Mar 2016 16:48:10 +0100 Subject: [PATCH 64/89] Swapping order of inserting block to chain and commiting to DB to avoid race conditions --- ethcore/src/client.rs | 12 +++++++----- sync/src/transaction_queue.rs | 4 ---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index d442a3d88..fdcd6c057 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -329,18 +329,14 @@ impl Client { bad_blocks.insert(header.hash()); continue; } - let closed_block = self.check_and_close_block(&block); if let Err(_) = closed_block { bad_blocks.insert(header.hash()); break; } - - // Insert block - let closed_block = closed_block.unwrap(); - self.chain.write().unwrap().insert_block(&block.bytes, closed_block.block().receipts().clone()); good_blocks.push(header.hash()); + // Are we committing an era? let ancient = if header.number() >= HISTORY { let n = header.number() - HISTORY; let chain = self.chain.read().unwrap(); @@ -350,10 +346,16 @@ impl Client { }; // Commit results + let closed_block = closed_block.unwrap(); + let receipts = closed_block.block().receipts().clone(); closed_block.drain() .commit(header.number(), &header.hash(), ancient) .expect("State DB commit failed."); + // And update the chain + self.chain.write().unwrap() + .insert_block(&block.bytes, receipts); + self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index f551fe435..83665dfda 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -240,7 +240,6 @@ impl TransactionQueue { let sender = transaction.sender(); let nonce = transaction.nonce(); - println!("Removing tx: {:?}", transaction.transaction); // Remove from future self.future.drop(&sender, &nonce); @@ -266,7 +265,6 @@ impl TransactionQueue { // Goes to future or is removed let order = self.current.drop(&sender, &k).unwrap(); if k >= current_nonce { - println!("Moving to future: {:?}", order); self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); } else { self.by_hash.remove(&order.hash); @@ -310,7 +308,6 @@ impl TransactionQueue { // remove also from priority and hash self.future.by_priority.remove(&order); // Put to current - println!("Moved: {:?}", order); let order = order.update_height(current_nonce.clone(), first_nonce); self.current.insert(address.clone(), current_nonce, order); current_nonce = current_nonce + U256::one(); @@ -331,7 +328,6 @@ impl TransactionQueue { .cloned() .map_or_else(|| fetch_nonce(&address), |n| n + U256::one()); - println!("Expected next: {:?}, got: {:?}", next_nonce, nonce); // Check height if nonce > next_nonce { let order = TransactionOrder::for_transaction(&tx, next_nonce); From d59972a9ac7f4cdda4006cdf300470dc07dd9c66 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 20:07:23 +0300 Subject: [PATCH 65/89] deserialization for uint generic --- util/bigint/src/uint.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/util/bigint/src/uint.rs b/util/bigint/src/uint.rs index a70997dc5..bd57e9d6d 100644 --- a/util/bigint/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -778,6 +778,35 @@ macro_rules! construct_uint { } } + impl serde::Deserialize for $name { + fn deserialize(deserializer: &mut D) -> Result<$name, D::Error> + where D: serde::Deserializer { + struct UintVisitor; + + impl serde::de::Visitor for UintVisitor { + type Value = $name; + + fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { + // 0x + len + if value.len() != 2 + $n_words / 8 { + return Err(serde::Error::custom("Invalid length.")); + } + + match $name::from_str(&value[2..]) { + Ok(val) => Ok(val), + Err(_) => { return Err(serde::Error::custom("Invalid length.")); } + } + } + + fn visit_string(&mut self, value: String) -> Result where E: serde::Error { + self.visit_str(value.as_ref()) + } + } + + deserializer.deserialize(UintVisitor) + } + } + impl From for $name { fn from(value: u64) -> $name { let mut ret = [0; $n_words]; From 2e3fb103abbc55d209fb4afbece9e933a54b1a7d Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 20:08:42 +0300 Subject: [PATCH 66/89] extended secret store operations --- util/src/keys/store.rs | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index f05b8dec4..bfb8e6c79 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -56,6 +56,17 @@ pub enum EncryptedHashMapError { InvalidValueFormat(FromBytesError), } +/// Error retrieving value from encrypted hashmap +#[derive(Debug)] +pub enum SigningError { + /// Account passed does not exist + NoAccount, + /// Account passed is not unlocked + AccountNotUnlocked, + /// Invalid secret in store + InvalidSecret +} + /// Represent service for storing encrypted arbitrary data pub struct SecretStore { directory: KeyDirectory, @@ -159,6 +170,26 @@ impl SecretStore { try!(self.directory.save(key_file)); Ok(address) } + + /// Signs message with unlocked account + pub fn sign(&self, account: &Address, message: &H256) -> Result { + let read_lock = self.unlocks.read().unwrap(); + let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); + match crypto::KeyPair::from_secret(unlock.secret) { + Ok(pair) => match pair.sign(message) { + Ok(signature) => Ok(signature), + Err(_) => Err(SigningError::InvalidSecret) + }, + Err(_) => Err(SigningError::InvalidSecret) + } + } + + /// Returns secret for unlocked account + pub fn account_secret(&self, account: &Address) -> Result { + let read_lock = self.unlocks.read().unwrap(); + let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); + Ok(unlock.secret as crypto::Secret) + } } fn derive_key_iterations(password: &str, salt: &H256, c: u32) -> (Bytes, Bytes) { @@ -423,6 +454,22 @@ mod tests { assert!(secret.is_ok()); } + #[test] + fn can_sign_data() { + let temp = RandomTempPath::create_dir(); + let address = { + let mut sstore = SecretStore::new_test(&temp); + sstore.new_account("334").unwrap() + }; + let signature = { + let sstore = SecretStore::new_test(&temp); + sstore.unlock_account(&address, "334").unwrap(); + sstore.sign(&address, &H256::random()).unwrap() + }; + + assert!(signature != x!(0)); + } + #[test] fn can_import_account() { use keys::directory::{KeyFileContent, KeyFileCrypto}; From c72c27b47ebb3bf5f4be3b4fb862f3f0ef29ff7f Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 20:09:21 +0300 Subject: [PATCH 67/89] client integration --- ethcore/src/client.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 6ae628f8b..070513c0a 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -36,6 +36,7 @@ use transaction::LocalizedTransaction; use extras::TransactionAddress; use filter::Filter; use log_entry::LocalizedLogEntry; +use util::keys::store::SecretStore; pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; @@ -202,7 +203,8 @@ pub struct Client where V: Verifier { sealing_block: Mutex>, author: RwLock
, extra_data: RwLock, - verifier: PhantomData + verifier: PhantomData, + secret_store: Arc>, } const HISTORY: u64 = 1000; @@ -238,6 +240,9 @@ impl Client where V: Verifier { let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); + let secret_store = Arc::new(RwLock::new(SecretStore::new())); + secret_store.write().unwrap().try_import_existing(); + Ok(Arc::new(Client { chain: chain, engine: engine, @@ -249,7 +254,8 @@ impl Client where V: Verifier { sealing_block: Mutex::new(None), author: RwLock::new(Address::new()), extra_data: RwLock::new(Vec::new()), - verifier: PhantomData + verifier: PhantomData, + secret_store: secret_store, })) } @@ -274,6 +280,11 @@ impl Client where V: Verifier { last_hashes } + /// Secret store (key manager) + pub fn secret_store(&self) -> &Arc> { + &self.secret_store + } + fn check_and_close_block(&self, block: &PreverifiedBlock) -> Result { let engine = self.engine.deref().deref(); let header = &block.header; From 3fa1776ecf43cb639c5ce688e85a5de1dc17c0f1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 19:11:44 +0100 Subject: [PATCH 68/89] Fixed sync stalling on a new block arriving while body request is pending --- sync/src/chain.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fd690e790..0b985f217 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -475,7 +475,7 @@ impl ChainSync { peer.latest_number = Some(header.number()); } // TODO: Decompose block and add to self.headers and self.bodies instead - if header.number == From::from(self.current_base_block() + 1) { + if header.number <= From::from(self.current_base_block() + 1) { match io.chain().import_block(block_rlp.as_raw().to_vec()) { Err(Error::Import(ImportError::AlreadyInChain)) => { trace!(target: "sync", "New block already in chain {:?}", h); @@ -484,7 +484,10 @@ impl ChainSync { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { - self.last_imported_block = Some(header.number); + if self.current_base_block() < header.number { + self.last_imported_block = Some(header.number); + self.remove_downloaded_blocks(header.number); + } trace!(target: "sync", "New block queued {:?}", h); }, Err(Error::Block(BlockError::UnknownParent(p))) => { From 8ff49c06dda93f241e46b99e93e9a954bec79e95 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 21:49:57 +0300 Subject: [PATCH 69/89] somehow that was not set to the right path --- sync/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 2ce65ca77..f10a772e3 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Ethcore Date: Fri, 4 Mar 2016 20:19:36 +0100 Subject: [PATCH 70/89] JournalDB can now operate in "archive" mode. --- util/src/journaldb.rs | 96 +++++++++++++++++++++++++++++++------------ util/src/overlaydb.rs | 2 +- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 5e6ca47c2..cb49134ee 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -25,7 +25,10 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; use std::env; /// Implementation of the HashDB trait for a disk-backed database with a memory overlay -/// and latent-removal semantics. +/// and, possibly, latent-removal semantics. +/// +/// If `counters` is `None`, then it behaves exactly like OverlayDB. If not it behaves +/// differently: /// /// Like OverlayDB, there is a memory overlay; `commit()` must be called in order to /// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect @@ -34,7 +37,7 @@ use std::env; pub struct JournalDB { overlay: MemoryDB, backing: Arc, - counters: Arc>>, + counters: Option>>>, } impl Clone for JournalDB { @@ -48,10 +51,11 @@ impl Clone for JournalDB { } // all keys must be at least 12 bytes -const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; +const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; -const DB_VERSION: u32 = 3; +const DB_VERSION : u32 = 3; +const DB_VERSION_NO_JOURNAL : u32 = 3 + 256; const PADDING : [u8; 10] = [ 0u8; 10 ]; @@ -59,25 +63,38 @@ impl JournalDB { /// Create a new instance from file pub fn new(path: &str) -> JournalDB { + Self::from_prefs(path, true) + } + + /// Create a new instance from file + pub fn from_prefs(path: &str, prefer_journal: bool) -> JournalDB { let opts = DatabaseConfig { prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix }; let backing = Database::open(&opts, path).unwrap_or_else(|e| { panic!("Error opening state db: {}", e); }); + let with_journal; if !backing.is_empty() { match backing.get(&VERSION_KEY).map(|d| d.map(|v| decode::(&v))) { - Ok(Some(DB_VERSION)) => {}, + Ok(Some(DB_VERSION)) => { with_journal = true; }, + Ok(Some(DB_VERSION_NO_JOURNAL)) => { with_journal = false; }, v => panic!("Incompatible DB version, expected {}, got {:?}", DB_VERSION, v) } } else { - backing.put(&VERSION_KEY, &encode(&DB_VERSION)).expect("Error writing version to database"); + backing.put(&VERSION_KEY, &encode(&(if prefer_journal { DB_VERSION } else { DB_VERSION_NO_JOURNAL }))).expect("Error writing version to database"); + with_journal = prefer_journal; } - let counters = JournalDB::read_counters(&backing); + + let counters = if with_journal { + Some(Arc::new(RwLock::new(JournalDB::read_counters(&backing)))) + } else { + None + }; JournalDB { overlay: MemoryDB::new(), backing: Arc::new(backing), - counters: Arc::new(RwLock::new(counters)), + counters: counters, } } @@ -94,9 +111,48 @@ impl JournalDB { self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() } + /// Commit all recent insert operations. + pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let have_counters = self.counters.is_some(); + if have_counters { + self.commit_with_counters(now, id, end) + } else { + self.commit_without_counters() + } + } + + /// Drain the overlay and place it into a batch for the DB. + fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> (usize, usize) { + let mut ret = 0usize; + let mut deletes = 0usize; + for i in overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc > 0 { + assert!(rc == 1); + batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); + ret += 1; + } + if rc < 0 { + assert!(rc == -1); + ret += 1; + deletes += 1; + } + } + (ret, deletes) + } + + /// Just commit the overlay into the backing DB. + fn commit_without_counters(&mut self) -> Result { + let batch = DBTransaction::new(); + let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); + try!(self.backing.write(batch)); + Ok(ret as u32) + + } + /// Commit all recent insert operations and historical removals from the old era /// to the backing database. - pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + fn commit_with_counters(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { // journal format: // [era, 0] => [ id, [insert_0, ...], [remove_0, ...] ] // [era, 1] => [ id, [insert_0, ...], [remove_0, ...] ] @@ -122,8 +178,8 @@ impl JournalDB { // record new commit's details. debug!("commit: #{} ({}), end era: {:?}", now, id, end); + let mut counters = self.counters.as_ref().unwrap().write().unwrap(); let batch = DBTransaction::new(); - let mut counters = self.counters.write().unwrap(); { let mut index = 0usize; let mut last; @@ -196,25 +252,11 @@ impl JournalDB { } // Commit overlay insertions - let mut ret = 0u32; - let mut deletes = 0usize; - for i in self.overlay.drain().into_iter() { - let (key, (value, rc)) = i; - if rc > 0 { - assert!(rc == 1); - batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); - ret += 1; - } - if rc < 0 { - assert!(rc == -1); - ret += 1; - deletes += 1; - } - } + let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); debug!("commit: Deleted {} nodes", deletes); - Ok(ret) + Ok(ret as u32) } diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index f4ed2d5d6..3c80f4148 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -146,7 +146,7 @@ impl OverlayDB { }) } - /// Get the refs and value of the given key. + /// Put the refs and value of the given key, possibly deleting it from the db. fn put_payload(&self, key: &H256, payload: (Bytes, u32)) -> bool { if payload.1 > 0 { let mut s = RlpStream::new_list(2); From bbbaffbc531571894b34f5c5a0de1e7eda453c60 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 21:06:28 +0100 Subject: [PATCH 71/89] "--archive" option for disabling the journal DB Fixes #579 --- ethcore/src/blockchain/blockchain.rs | 3 +++ ethcore/src/client.rs | 5 +++-- parity/main.rs | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index f412a8240..e79f1668c 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -40,6 +40,8 @@ pub struct BlockChainConfig { pub pref_cache_size: usize, /// Maximum cache size in bytes. pub max_cache_size: usize, + /// Prefer journal rather than archive. + pub prefer_journal: bool, } impl Default for BlockChainConfig { @@ -47,6 +49,7 @@ impl Default for BlockChainConfig { BlockChainConfig { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, + prefer_journal: false, } } } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 422f1eaa2..87f2c9e96 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -212,7 +212,8 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); + dir.push(format!("v{}-sec-pruned-{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "journal" } else { "archive" })); + let pj = config.blockchain.prefer_journal; let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); @@ -220,7 +221,7 @@ impl Client { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::new(state_path.to_str().unwrap()); + let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), pj); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/parity/main.rs b/parity/main.rs index b991f36cd..f1a11229f 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -65,6 +65,7 @@ Usage: Options: --chain CHAIN Specify the blockchain type. CHAIN may be either a JSON chain specification file or frontier, mainnet, morden, or testnet [default: frontier]. + --archive Client should not prune the state/storage trie. -d --db-path PATH Specify the database & configuration directory path [default: $HOME/.parity] --keys-path PATH Specify the path for JSON key files to be found [default: $HOME/.web3/keys] @@ -102,6 +103,7 @@ struct Args { flag_chain: String, flag_db_path: String, flag_keys_path: String, + flag_archive: bool, flag_no_bootstrap: bool, flag_listen_address: String, flag_public_address: Option, @@ -311,6 +313,7 @@ impl Configuration { let mut client_config = ClientConfig::default(); client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; + client_config.blockchain.prefer_journal = !self.args.flag_archive; client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); From bc018faedcffdcea398cee7848a4e2d3a64ce78a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 21:17:42 +0100 Subject: [PATCH 72/89] Avoid forcing a resync for the pre-existing journaldbs. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 87f2c9e96..4c9e76c76 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -212,7 +212,7 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned-{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "journal" } else { "archive" })); + dir.push(format!("v{}-sec-pruned{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "" } else { "-archive" })); let pj = config.blockchain.prefer_journal; let path = dir.as_path(); let gb = spec.genesis_block(); From f028ff1d40f95c5a5f8f0ab3cf85a9df5873b57c Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 21:52:03 +0100 Subject: [PATCH 73/89] Use same BlockChainInfo for propagation --- sync/src/chain.rs | 56 +++++++++++++++++---------------------- sync/src/tests/helpers.rs | 2 +- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index fd690e790..cce8b12cf 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -33,7 +33,7 @@ use util::*; use std::mem::{replace}; use ethcore::views::{HeaderView}; use ethcore::header::{BlockNumber, Header as BlockHeader}; -use ethcore::client::{BlockChainClient, BlockStatus, BlockId}; +use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo}; use range_collection::{RangeCollection, ToUsize, FromUsize}; use ethcore::error::*; use ethcore::block::Block; @@ -1156,9 +1156,7 @@ impl ChainSync { } /// returns peer ids that have less blocks than our chain - fn get_lagging_peers(&mut self, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { - let chain = io.chain(); - let chain_info = chain.chain_info(); + fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { let latest_hash = chain_info.best_block_hash; let latest_number = chain_info.best_block_number; self.peers.iter_mut().filter_map(|(&id, ref mut peer_info)| @@ -1177,9 +1175,9 @@ impl ChainSync { } /// propagates latest block to lagging peers - fn propagate_blocks(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { + fn propagate_blocks(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> usize { let updated_peers = { - let lagging_peers = self.get_lagging_peers(io); + let lagging_peers = self.get_lagging_peers(chain_info, io); // sqrt(x)/x scaled to max u32 let fraction = (self.peers.len() as f64).powf(-0.5).mul(u32::max_value() as f64).round() as u32; @@ -1196,30 +1194,30 @@ impl ChainSync { for peer_id in updated_peers { let rlp = ChainSync::create_latest_block_rlp(io.chain()); self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); - self.peers.get_mut(&peer_id).unwrap().latest_hash = local_best.clone(); - self.peers.get_mut(&peer_id).unwrap().latest_number = Some(best_number); + self.peers.get_mut(&peer_id).unwrap().latest_hash = chain_info.best_block_hash.clone(); + self.peers.get_mut(&peer_id).unwrap().latest_number = Some(chain_info.best_block_number); sent = sent + 1; } sent } /// propagates new known hashes to all peers - fn propagate_new_hashes(&mut self, local_best: &H256, best_number: BlockNumber, io: &mut SyncIo) -> usize { - let updated_peers = self.get_lagging_peers(io); + fn propagate_new_hashes(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> usize { + let updated_peers = self.get_lagging_peers(chain_info, io); let mut sent = 0; - let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(local_best.clone())).unwrap()).parent_hash(); + let last_parent = HeaderView::new(&io.chain().block_header(BlockId::Hash(chain_info.best_block_hash.clone())).unwrap()).parent_hash(); for (peer_id, peer_number) in updated_peers { let mut peer_best = self.peers.get(&peer_id).unwrap().latest_hash.clone(); - if best_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { + if chain_info.best_block_number - peer_number > MAX_PEERS_PROPAGATION as BlockNumber { // If we think peer is too far behind just send one latest hash peer_best = last_parent.clone(); } - sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &local_best) { + sent = sent + match ChainSync::create_new_hashes_rlp(io.chain(), &peer_best, &chain_info.best_block_hash) { Some(rlp) => { { let peer = self.peers.get_mut(&peer_id).unwrap(); - peer.latest_hash = local_best.clone(); - peer.latest_number = Some(best_number); + peer.latest_hash = chain_info.best_block_hash.clone(); + peer.latest_number = Some(chain_info.best_block_number); } self.send_packet(io, peer_id, NEW_BLOCK_HASHES_PACKET, rlp); 1 @@ -1239,8 +1237,8 @@ impl ChainSync { pub fn chain_blocks_verified(&mut self, io: &mut SyncIo) { let chain = io.chain().chain_info(); if (((chain.best_block_number as i64) - (self.last_send_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let blocks = self.propagate_blocks(&chain.best_block_hash, chain.best_block_number, io); - let hashes = self.propagate_new_hashes(&chain.best_block_hash, chain.best_block_number, io); + let blocks = self.propagate_blocks(&chain, io); + let hashes = self.propagate_new_hashes(&chain, io); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); } @@ -1390,9 +1388,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); + let chain_info = client.chain_info(); let io = TestIo::new(&mut client, &mut queue, None); - let lagging_peers = sync.get_lagging_peers(&io); + let lagging_peers = sync.get_lagging_peers(&chain_info, &io); assert_eq!(1, lagging_peers.len()) } @@ -1420,11 +1419,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagate_new_hashes(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_new_hashes(&chain_info, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1440,11 +1438,9 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - - let peer_count = sync.propagate_blocks(&best_hash, best_number, &mut io); + let peer_count = sync.propagate_blocks(&chain_info, &mut io); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1546,11 +1542,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagate_new_hashes(&best_hash, best_number, &mut io); + sync.propagate_new_hashes(&chain_info, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_hashes(&mut io, 0, &UntrustedRlp::new(&data)); @@ -1565,11 +1560,10 @@ mod tests { client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - let best_hash = client.chain_info().best_block_hash.clone(); - let best_number = client.chain_info().best_block_number; + let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagate_blocks(&best_hash, best_number, &mut io); + sync.propagate_blocks(&chain_info, &mut io); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &UntrustedRlp::new(&data)); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index e9c5f0edc..e170a4a85 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -105,7 +105,7 @@ impl BlockChainClient for TestBlockChainClient { Some(U256::zero()) } - fn block_hash(&self, id: BlockId) -> Option { + fn block_hash(&self, _id: BlockId) -> Option { unimplemented!(); } From 182aec2f9463f0c96b6bec1022e5328935785248 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 22:01:36 +0100 Subject: [PATCH 74/89] Fixed potential deadlock on startup --- util/src/network/host.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 42e8ff93d..f2cc9fe48 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -400,7 +400,8 @@ impl Host where Message: Send + Sync + Clone { // public_endpoint in host info contains local adderss at this point let listen_address = self.info.read().unwrap().public_endpoint.address.clone(); let udp_port = self.info.read().unwrap().config.udp_port.unwrap_or(listen_address.port()); - let public_endpoint = match self.info.read().unwrap().config.public_address { + let public_address = self.info.read().unwrap().config.public_address.clone(); + let public_endpoint = match public_address { None => { let public_address = select_public_address(listen_address.port()); let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; From 559e01ea84beb7ff46c67868c69d6e817ba1f986 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 22:54:59 +0100 Subject: [PATCH 75/89] Review remarks resolved. --- ethcore/src/client.rs | 19 +++++++++++++++---- parity/main.rs | 2 +- util/src/journaldb.rs | 9 ++++----- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 4c9e76c76..064b749a8 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -76,12 +76,24 @@ pub enum BlockStatus { } /// Client configuration. Includes configs for all sub-systems. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct ClientConfig { /// Block queue configuration. pub queue: BlockQueueConfig, /// Blockchain configuration. pub blockchain: BlockChainConfig, + /// Prefer journal rather than archive. + pub prefer_journal: bool, +} + +impl Default for ClientConfig { + fn default() -> ClientConfig { + ClientConfig { + queue: Default::default(), + blockchain: Default::default(), + prefer_journal: false, + } + } } /// Information about the blockchain gathered together. @@ -212,8 +224,7 @@ impl Client { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning - dir.push(format!("v{}-sec-pruned{}", CLIENT_DB_VER_STR, if config.blockchain.prefer_journal { "" } else { "-archive" })); - let pj = config.blockchain.prefer_journal; + dir.push(format!("v{}-sec-{}", CLIENT_DB_VER_STR, if config.prefer_journal { "pruned" } else { "archive" })); let path = dir.as_path(); let gb = spec.genesis_block(); let chain = Arc::new(RwLock::new(BlockChain::new(config.blockchain, &gb, path))); @@ -221,7 +232,7 @@ impl Client { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), pj); + let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal); if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } diff --git a/parity/main.rs b/parity/main.rs index f1a11229f..3f4243a0a 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -313,7 +313,7 @@ impl Configuration { let mut client_config = ClientConfig::default(); client_config.blockchain.pref_cache_size = self.args.flag_cache_pref_size; client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; - client_config.blockchain.prefer_journal = !self.args.flag_archive; + client_config.prefer_journal = !self.args.flag_archive; client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); let client = service.client().clone(); diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index cb49134ee..b20934397 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -147,7 +147,6 @@ impl JournalDB { let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); Ok(ret as u32) - } /// Commit all recent insert operations and historical removals from the old era @@ -177,7 +176,7 @@ impl JournalDB { // and the key is safe to delete. // record new commit's details. - debug!("commit: #{} ({}), end era: {:?}", now, id, end); + trace!("commit: #{} ({}), end era: {:?}", now, id, end); let mut counters = self.counters.as_ref().unwrap().write().unwrap(); let batch = DBTransaction::new(); { @@ -248,14 +247,14 @@ impl JournalDB { try!(batch.delete(&h)); deletes += 1; } - debug!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); + trace!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); } // Commit overlay insertions let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); - debug!("commit: Deleted {} nodes", deletes); + trace!("commit: Deleted {} nodes", deletes); Ok(ret as u32) } @@ -304,7 +303,7 @@ impl JournalDB { era -= 1; } } - debug!("Recovered {} counters", res.len()); + trace!("Recovered {} counters", res.len()); res } } From 96617533c8d41ac8726bd17e7de8f3f98b9178d2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 22:57:44 +0100 Subject: [PATCH 76/89] Remove unneeded field. --- ethcore/src/blockchain/blockchain.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e79f1668c..f412a8240 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -40,8 +40,6 @@ pub struct BlockChainConfig { pub pref_cache_size: usize, /// Maximum cache size in bytes. pub max_cache_size: usize, - /// Prefer journal rather than archive. - pub prefer_journal: bool, } impl Default for BlockChainConfig { @@ -49,7 +47,6 @@ impl Default for BlockChainConfig { BlockChainConfig { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, - prefer_journal: false, } } } From 098a6ad2cc3c33810e9036d0a29841a3124a7e18 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:09:05 +0100 Subject: [PATCH 77/89] Reset `HISTORY`. --- ethcore/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index bd8c5175b..fb46c81b1 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -207,7 +207,7 @@ pub struct Client where V: Verifier { secret_store: Arc>, } -const HISTORY: u64 = 30; +const HISTORY: u64 = 1000; const CLIENT_DB_VER_STR: &'static str = "4.0"; impl Client { From 86c34c7d10b36f39437289fa439613ae73d3f44a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:29:56 +0100 Subject: [PATCH 78/89] Remove "fix". --- ethcore/src/block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index d05ce51b9..68f647e37 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -220,8 +220,8 @@ impl<'x> OpenBlock<'x> { /// NOTE Will check chain constraints and the uncle number but will NOT check /// that the header itself is actually valid. pub fn push_uncle(&mut self, valid_uncle_header: Header) -> Result<(), BlockError> { - if self.block.base.uncles.len() > self.engine.maximum_uncle_count() { - return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len()})); + if self.block.base.uncles.len() + 1 > self.engine.maximum_uncle_count() { + return Err(BlockError::TooManyUncles(OutOfBounds{min: None, max: Some(self.engine.maximum_uncle_count()), found: self.block.base.uncles.len() + 1})); } // TODO: check number // TODO: check not a direct ancestor (use last_hashes for that) From 8f0005617171f6b7c71c9a2f10fc5e48821248b1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Mar 2016 23:43:59 +0100 Subject: [PATCH 79/89] Avoid sealing unnecessarily. --- ethcore/src/client.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index fb46c81b1..845702285 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -17,6 +17,7 @@ //! Blockchain database client. use std::marker::PhantomData; +use std::sync::atomic::AtomicBool; use util::*; use util::panics::*; use blockchain::{BlockChain, BlockProvider}; @@ -200,6 +201,7 @@ pub struct Client where V: Verifier { panic_handler: Arc, // for sealing... + sealing_enabled: AtomicBool, sealing_block: Mutex>, author: RwLock
, extra_data: RwLock, @@ -251,6 +253,7 @@ impl Client where V: Verifier { report: RwLock::new(Default::default()), import_lock: Mutex::new(()), panic_handler: panic_handler, + sealing_enabled: AtomicBool::new(false), sealing_block: Mutex::new(None), author: RwLock::new(Address::new()), extra_data: RwLock::new(Vec::new()), @@ -398,7 +401,7 @@ impl Client where V: Verifier { } } - if self.chain_info().best_block_hash != original_best { + if self.chain_info().best_block_hash != original_best && self.sealing_enabled.load(atomic::Ordering::Relaxed) { self.prepare_sealing(); } @@ -481,7 +484,7 @@ impl Client where V: Verifier { self.extra_data() ); - self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().foreach(|h| { b.push_uncle(h).unwrap(); }); + self.chain.read().unwrap().find_uncle_headers(&h, self.engine.deref().deref().maximum_uncle_age()).unwrap().into_iter().take(self.engine.deref().deref().maximum_uncle_count()).foreach(|h| { b.push_uncle(h).unwrap(); }); // TODO: push transactions. @@ -493,6 +496,8 @@ impl Client where V: Verifier { /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. pub fn sealing_block(&self) -> &Mutex> { if self.sealing_block.lock().unwrap().is_none() { + self.sealing_enabled.store(true, atomic::Ordering::Relaxed); + // TODO: Above should be on a timer that resets after two blocks have arrived without being asked for. self.prepare_sealing(); } &self.sealing_block From 2d6738fcde45f32d3b5b95e8e6b584f928864f83 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 4 Mar 2016 23:53:57 +0100 Subject: [PATCH 80/89] Additional logging and assert --- util/src/journaldb.rs | 45 ++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index b20934397..e3dd2bbfd 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -122,29 +122,29 @@ impl JournalDB { } /// Drain the overlay and place it into a batch for the DB. - fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> (usize, usize) { - let mut ret = 0usize; + fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize { + let mut inserts = 0usize; let mut deletes = 0usize; for i in overlay.drain().into_iter() { let (key, (value, rc)) = i; if rc > 0 { assert!(rc == 1); batch.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); - ret += 1; + inserts += 1; } if rc < 0 { assert!(rc == -1); - ret += 1; deletes += 1; } } - (ret, deletes) + trace!("commit: Inserted {}, Deleted {} nodes", inserts, deletes); + inserts + deletes } /// Just commit the overlay into the backing DB. fn commit_without_counters(&mut self) -> Result { let batch = DBTransaction::new(); - let (ret, _) = Self::batch_overlay_insertions(&mut self.overlay, &batch); + let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); Ok(ret as u32) } @@ -183,14 +183,23 @@ impl JournalDB { let mut index = 0usize; let mut last; - while try!(self.backing.get({ - let mut r = RlpStream::new_list(3); - r.append(&now); - r.append(&index); - r.append(&&PADDING[..]); - last = r.drain(); - &last - })).is_some() { + while { + let record = try!(self.backing.get({ + let mut r = RlpStream::new_list(3); + r.append(&now); + r.append(&index); + r.append(&&PADDING[..]); + last = r.drain(); + &last + })); + match record { + Some(r) => { + assert!(&Rlp::new(&r).val_at::(0) != id); + true + }, + None => false, + } + } { index += 1; } @@ -236,6 +245,7 @@ impl JournalDB { trace!("Purging nodes inserted in non-canon: {:?}", inserts); to_remove.append(&mut inserts); } + trace!("commit: Delete journal for time #{}.{}: {}, (canon was {}): {} entries", end_era, index, rlp.val_at::(0), canon_id, to_remove.len()); try!(batch.delete(&last)); index += 1; } @@ -243,18 +253,17 @@ impl JournalDB { let canon_inserts = canon_inserts.drain(..).collect::>(); // Purge removed keys if they are not referenced and not re-inserted in the canon commit let mut deletes = 0; + trace!("Purging filtered notes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); for h in to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)) { try!(batch.delete(&h)); deletes += 1; } - trace!("commit: Delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, deletes); + trace!("Total nodes purged: {}", deletes); } // Commit overlay insertions - let (ret, deletes) = Self::batch_overlay_insertions(&mut self.overlay, &batch); - + let ret = Self::batch_overlay_insertions(&mut self.overlay, &batch); try!(self.backing.write(batch)); - trace!("commit: Deleted {} nodes", deletes); Ok(ret as u32) } From a4640beb2c766964d5fa905fba0d3eacfb35725c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 5 Mar 2016 00:00:43 +0100 Subject: [PATCH 81/89] Typo --- util/src/journaldb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index e3dd2bbfd..01e53f819 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -253,7 +253,7 @@ impl JournalDB { let canon_inserts = canon_inserts.drain(..).collect::>(); // Purge removed keys if they are not referenced and not re-inserted in the canon commit let mut deletes = 0; - trace!("Purging filtered notes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); + trace!("Purging filtered nodes: {:?}", to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)).collect::>()); for h in to_remove.iter().filter(|h| !counters.contains_key(h) && !canon_inserts.contains(h)) { try!(batch.delete(&h)); deletes += 1; From 5ad577301420c092454c6268ca3f7a38dbb5ff44 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Mar 2016 10:45:05 +0100 Subject: [PATCH 82/89] verifier improvements --- ethcore/src/client.rs | 4 ++-- ethcore/src/verification/canon_verifier.rs | 6 ++++++ ethcore/src/verification/noop_verifier.rs | 6 ++++++ ethcore/src/verification/verification.rs | 2 +- ethcore/src/verification/verifier.rs | 3 +++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c40ac2ab8..858185873 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -233,7 +233,7 @@ impl Client { impl Client where V: Verifier { /// Create a new client with given spec and DB path and custom verifier. - pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result, Error> { + pub fn new_with_verifier(config: ClientConfig, spec: Spec, path: &Path, message_channel: IoChannel ) -> Result>, Error> { let mut dir = path.to_path_buf(); dir.push(H64::from(spec.genesis_header().hash()).hex()); //TODO: sec/fat: pruned/full versioning @@ -312,7 +312,7 @@ impl Client where V: Verifier { } // Verify Block Family - let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); + let verify_family_result = V::verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); if let Err(e) = verify_family_result { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); diff --git a/ethcore/src/verification/canon_verifier.rs b/ethcore/src/verification/canon_verifier.rs index 0d9cbc6b6..30e368f1b 100644 --- a/ethcore/src/verification/canon_verifier.rs +++ b/ethcore/src/verification/canon_verifier.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; use super::Verifier; @@ -22,6 +24,10 @@ use super::verification; pub struct CanonVerifier; impl Verifier for CanonVerifier { + fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> { + verification::verify_block_family(header, bytes, engine, bc) + } + fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> { verification::verify_block_final(expected, got) } diff --git a/ethcore/src/verification/noop_verifier.rs b/ethcore/src/verification/noop_verifier.rs index 8dfd64771..ae2a153fe 100644 --- a/ethcore/src/verification/noop_verifier.rs +++ b/ethcore/src/verification/noop_verifier.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; use super::Verifier; @@ -21,6 +23,10 @@ use super::Verifier; pub struct NoopVerifier; impl Verifier for NoopVerifier { + fn verify_block_family(_header: &Header, _bytes: &[u8], _engine: &Engine, _bc: &BlockProvider) -> Result<(), Error> { + Ok(()) + } + fn verify_block_final(_expected: &Header, _got: &Header) -> Result<(), Error> { Ok(()) } diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 761e2e8cd..ed3db3791 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -78,7 +78,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> } /// Phase 3 verification. Check block information against parent and uncles. -pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BC) -> Result<(), Error> where BC: BlockProvider { +pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> { // TODO: verify timestamp let parent = try!(bc.block_header(&header.parent_hash).ok_or_else(|| Error::from(BlockError::UnknownParent(header.parent_hash.clone())))); try!(verify_parent(&header, &parent)); diff --git a/ethcore/src/verification/verifier.rs b/ethcore/src/verification/verifier.rs index 0ffbf3bdd..cc5edce29 100644 --- a/ethcore/src/verification/verifier.rs +++ b/ethcore/src/verification/verifier.rs @@ -14,10 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use blockchain::BlockProvider; +use engine::Engine; use error::Error; use header::Header; /// Should be used to verify blocks. pub trait Verifier: Send + Sync { + fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>; fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>; } From 1d04a7b8f98614c4a883585d0b84a35a1a469853 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 13:11:54 +0300 Subject: [PATCH 83/89] changing warning to trace --- util/src/keys/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index bfb8e6c79..625d6fd8f 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -105,7 +105,7 @@ impl SecretStore { import_path.push(".ethereum"); import_path.push("keystore"); if let Err(e) = geth_import::import_geth_keys(self, &import_path) { - warn!(target: "sstore", "Error retrieving geth keys: {:?}", e) + trace!(target: "sstore", "Geth key not imported: {:?}", e); } } From 8a13e87cbeb0905077a25092f088a54482ab2e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 5 Mar 2016 11:30:09 +0100 Subject: [PATCH 84/89] Renaming BlocksWith helper to EachBlockWith --- sync/src/chain.rs | 34 +++++++++++++-------------- sync/src/tests/chain.rs | 48 +++++++++++++++++++-------------------- sync/src/tests/helpers.rs | 10 ++++---- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 983ce62c3..e8ad81a3a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -491,7 +491,7 @@ impl ChainSync { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { - if self.current_base_block() < header.number { + if self.current_base_block() < header.number { self.last_imported_block = Some(header.number); self.remove_downloaded_blocks(header.number); } @@ -1433,7 +1433,7 @@ mod tests { #[test] fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let chain_info = client.chain_info(); @@ -1447,7 +1447,7 @@ mod tests { #[test] fn calculates_tree_for_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(15, BlocksWith::Uncle); + client.add_blocks(15, EachBlockWith::Uncle); let start = client.block_hash_delta_minus(4); let end = client.block_hash_delta_minus(2); @@ -1464,7 +1464,7 @@ mod tests { #[test] fn sends_new_hashes_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1483,7 +1483,7 @@ mod tests { #[test] fn sends_latest_block_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1501,7 +1501,7 @@ mod tests { #[test] fn handles_peer_new_block_mallformed() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let block_data = get_dummy_block(11, client.chain_info().best_block_hash); @@ -1519,7 +1519,7 @@ mod tests { #[test] fn handles_peer_new_block() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); @@ -1537,7 +1537,7 @@ mod tests { #[test] fn handles_peer_new_block_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1553,7 +1553,7 @@ mod tests { #[test] fn handles_peer_new_hashes() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1569,7 +1569,7 @@ mod tests { #[test] fn handles_peer_new_hashes_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, BlocksWith::Uncle); + client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1587,7 +1587,7 @@ mod tests { #[test] fn hashes_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1605,7 +1605,7 @@ mod tests { #[test] fn block_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1622,9 +1622,9 @@ mod tests { fn should_add_transactions_to_queue() { // given let mut client = TestBlockChainClient::new(); - client.add_blocks(98, BlocksWith::Uncle); - client.add_blocks(1, BlocksWith::UncleAndTransaction); - client.add_blocks(1, BlocksWith::Transaction); + client.add_blocks(98, EachBlockWith::Uncle); + client.add_blocks(1, EachBlockWith::UncleAndTransaction); + client.add_blocks(1, EachBlockWith::Transaction); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; @@ -1648,7 +1648,7 @@ mod tests { #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1672,7 +1672,7 @@ mod tests { #[test] fn returns_requested_block_headers_reverse() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, BlocksWith::Uncle); + client.add_blocks(100, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index b388f508d..58f50916e 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -24,8 +24,8 @@ use super::helpers::*; fn two_peers() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); assert_eq!(net.peer(0).chain.blocks.read().unwrap().deref(), net.peer(1).chain.blocks.read().unwrap().deref()); @@ -35,8 +35,8 @@ fn two_peers() { fn status_after_sync() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync(); let status = net.peer(0).sync.status(); assert_eq!(status.state, SyncState::Idle); @@ -45,8 +45,8 @@ fn status_after_sync() { #[test] fn takes_few_steps() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(100, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(100, EachBlockWith::Uncle); let total_steps = net.sync(); assert!(total_steps < 7); } @@ -56,7 +56,7 @@ fn empty_blocks() { ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { - let with = if n % 2 == 0 { BlocksWith::Nothing } else { BlocksWith::Uncle }; + let with = if n % 2 == 0 { EachBlockWith::Nothing } else { EachBlockWith::Uncle }; net.peer_mut(1).chain.add_blocks(5, with.clone()); net.peer_mut(2).chain.add_blocks(5, with); } @@ -69,14 +69,14 @@ fn empty_blocks() { fn forked() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(0).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(300, BlocksWith::Uncle); - net.peer_mut(0).chain.add_blocks(100, BlocksWith::Nothing); //fork - net.peer_mut(1).chain.add_blocks(200, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(200, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(100, BlocksWith::Uncle); //fork between 1 and 2 - net.peer_mut(2).chain.add_blocks(10, BlocksWith::Nothing); + net.peer_mut(0).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(300, EachBlockWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Nothing); //fork + net.peer_mut(1).chain.add_blocks(200, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(200, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, EachBlockWith::Nothing); // peer 1 has the best chain of 601 blocks let peer1_chain = net.peer(1).chain.numbers.read().unwrap().clone(); net.sync(); @@ -88,8 +88,8 @@ fn forked() { #[test] fn restart() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, BlocksWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); net.sync_steps(8); @@ -110,8 +110,8 @@ fn status_empty() { #[test] fn status_packet() { let mut net = TestNet::new(2); - net.peer_mut(0).chain.add_blocks(100, BlocksWith::Uncle); - net.peer_mut(1).chain.add_blocks(1, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(1, EachBlockWith::Uncle); net.start(); @@ -124,10 +124,10 @@ fn status_packet() { #[test] fn propagate_hashes() { let mut net = TestNet::new(6); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -150,10 +150,10 @@ fn propagate_hashes() { #[test] fn propagate_blocks() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.sync(); - net.peer_mut(0).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -165,7 +165,7 @@ fn propagate_blocks() { #[test] fn restart_on_malformed_block() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, BlocksWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); net.peer_mut(1).chain.corrupt_block(6); net.sync_steps(10); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index ca7776bf3..5b53ad90b 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -35,7 +35,7 @@ pub struct TestBlockChainClient { } #[derive(Clone)] -pub enum BlocksWith { +pub enum EachBlockWith { Nothing, Uncle, Transaction, @@ -52,12 +52,12 @@ impl TestBlockChainClient { last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), }; - client.add_blocks(1, BlocksWith::Nothing); // add genesis block + client.add_blocks(1, EachBlockWith::Nothing); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } - pub fn add_blocks(&mut self, count: usize, with: BlocksWith) { + pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { let len = self.numbers.read().unwrap().len(); for n in len..(len + count) { let mut header = BlockHeader::new(); @@ -65,7 +65,7 @@ impl TestBlockChainClient { header.parent_hash = self.last_hash.read().unwrap().clone(); header.number = n as BlockNumber; let uncles = match with { - BlocksWith::Uncle | BlocksWith::UncleAndTransaction => { + EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => { let mut uncles = RlpStream::new_list(1); let mut uncle_header = BlockHeader::new(); uncle_header.difficulty = From::from(n); @@ -78,7 +78,7 @@ impl TestBlockChainClient { _ => RlpStream::new_list(0) }; let txs = match with { - BlocksWith::Transaction | BlocksWith::UncleAndTransaction => { + EachBlockWith::Transaction | EachBlockWith::UncleAndTransaction => { let mut txs = RlpStream::new_list(1); let keypair = KeyPair::create().unwrap(); let tx = Transaction { From 9e5f8d44342ccfb93a87c46f82498a2b974c6621 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Mar 2016 11:35:44 +0100 Subject: [PATCH 85/89] build on rust stable --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d2349dae..7213b8f09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: allow_failures: - rust: nightly include: + - rust: stable + env: FEATURES="--features travis-beta" KCOV_FEATURES="" TARGETS="-p ethash -p ethcore-util -p ethcore -p ethsync -p ethcore-rpc -p parity" ARCHIVE_SUFFIX="-${TRAVIS_OS_NAME}-${TRAVIS_TAG}" - rust: beta env: FEATURES="--features travis-beta" KCOV_FEATURES="" TARGETS="-p ethash -p ethcore-util -p ethcore -p ethsync -p ethcore-rpc -p parity" ARCHIVE_SUFFIX="-${TRAVIS_OS_NAME}-${TRAVIS_TAG}" - rust: nightly @@ -52,7 +54,7 @@ after_success: | ./kcov-master/tmp/usr/local/bin/kcov --coveralls-id=${TRAVIS_JOB_ID} --exclude-pattern /usr/,/.cargo,/root/.multirust target/kcov target/debug/parity-* && [ $TRAVIS_BRANCH = master ] && [ $TRAVIS_PULL_REQUEST = false ] && - [ $TRAVIS_RUST_VERSION = beta ] && + [ $TRAVIS_RUST_VERSION = stable ] && cargo doc --no-deps --verbose ${KCOV_FEATURES} ${TARGETS} && echo '' > target/doc/index.html && pip install --user ghp-import && From b9a6a70cede04e276aaa9e1ea38ee5cef17d3133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 5 Mar 2016 11:37:19 +0100 Subject: [PATCH 86/89] Renaming bad blocks as retracted --- ethcore/src/client.rs | 2 +- ethcore/src/service.rs | 2 +- sync/src/chain.rs | 11 ++++++----- sync/src/lib.rs | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index ef0356d3e..878bacce9 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -410,7 +410,7 @@ impl Client where V: Verifier { if !good_blocks.is_empty() && block_queue.queue_info().is_empty() { io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { good: good_blocks, - bad: bad_blocks, + retracted: bad_blocks, })).unwrap(); } } diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 756d02407..a80adb0ba 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -30,7 +30,7 @@ pub enum SyncMessage { /// Hashes of blocks imported to blockchain good: Vec, /// Hashes of blocks not imported to blockchain - bad: Vec, + retracted: Vec, }, /// A block is ready BlockVerified, diff --git a/sync/src/chain.rs b/sync/src/chain.rs index e8ad81a3a..ddf30854a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1265,10 +1265,11 @@ impl ChainSync { } /// called when block is imported to chain, updates transactions queue - pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], bad: &[H256]) { + pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], retracted: &[H256]) { fn fetch_transactions(chain: &BlockChainClient, hash: &H256) -> Vec { let block = chain .block(BlockId::Hash(hash.clone())) + // Client should send message after commit to db and inserting to chain. .expect("Expected in-chain blocks."); let block = BlockView::new(&block); block.transactions() @@ -1277,14 +1278,14 @@ impl ChainSync { let chain = io.chain(); let good = good.par_iter().map(|h| fetch_transactions(chain, h)); - let bad = bad.par_iter().map(|h| fetch_transactions(chain, h)); + let retracted = retracted.par_iter().map(|h| fetch_transactions(chain, h)); good.for_each(|txs| { let mut transaction_queue = self.transaction_queue.lock().unwrap(); let hashes = txs.iter().map(|tx| tx.hash()).collect::>(); transaction_queue.remove_all(&hashes, |a| chain.nonce(a)); }); - bad.for_each(|txs| { + retracted.for_each(|txs| { // populate sender for tx in &txs { let _sender = tx.sender(); @@ -1628,7 +1629,7 @@ mod tests { let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let good_blocks = vec![client.block_hash_delta_minus(2)]; - let bad_blocks = vec![client.block_hash_delta_minus(1)]; + let retracted_blocks = vec![client.block_hash_delta_minus(1)]; let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1637,7 +1638,7 @@ mod tests { sync.chain_new_blocks(&io, &[], &good_blocks); assert_eq!(sync.transaction_queue.lock().unwrap().status().future, 0); assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); - sync.chain_new_blocks(&io, &good_blocks, &bad_blocks); + sync.chain_new_blocks(&io, &good_blocks, &retracted_blocks); // then let status = sync.transaction_queue.lock().unwrap().status(); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 44f3f02e0..d67a09f3b 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -157,9 +157,9 @@ impl NetworkProtocolHandler for EthSync { SyncMessage::BlockVerified => { self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); }, - SyncMessage::NewChainBlocks { ref good, ref bad } => { + SyncMessage::NewChainBlocks { ref good, ref retracted } => { let sync_io = NetSyncIo::new(io, self.chain.deref()); - self.sync.write().unwrap().chain_new_blocks(&sync_io, good, bad); + self.sync.write().unwrap().chain_new_blocks(&sync_io, good, retracted); } } } From cfbaa2d6e99cecbd499fa7fc0eefc510e6338e84 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 14:25:46 +0300 Subject: [PATCH 87/89] fixed namespaces --- util/benches/bigint.rs | 2 +- util/benches/rlp.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index fc41ab628..575164cb6 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -28,7 +28,7 @@ extern crate ethcore_util; extern crate rand; use test::{Bencher, black_box}; -use ethcore_util::uint::*; +use ethcore_util::numbers::*; #[bench] fn u256_add(b: &mut Bencher) { diff --git a/util/benches/rlp.rs b/util/benches/rlp.rs index e94cb3635..4a983f369 100644 --- a/util/benches/rlp.rs +++ b/util/benches/rlp.rs @@ -28,7 +28,7 @@ extern crate ethcore_util; use test::Bencher; use std::str::FromStr; use ethcore_util::rlp::*; -use ethcore_util::uint::U256; +use ethcore_util::numbers::U256; #[bench] fn bench_stream_u64_value(b: &mut Bencher) { From d330f0b7b7fa5db1b5891d7c1e4e61136603fed5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 5 Mar 2016 12:53:54 +0100 Subject: [PATCH 88/89] Revert "Transaction Queue integration" --- Cargo.lock | 19 ------ ethcore/src/client.rs | 21 ++----- ethcore/src/service.rs | 2 +- sync/Cargo.toml | 1 - sync/src/chain.rs | 107 ++++++---------------------------- sync/src/lib.rs | 14 ++--- sync/src/tests/chain.rs | 51 ++++++++-------- sync/src/tests/helpers.rs | 61 +++++-------------- sync/src/transaction_queue.rs | 23 +++++--- 9 files changed, 80 insertions(+), 219 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 510e69b59..55ed996ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,14 +146,6 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "deque" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "docopt" version = "0.6.78" @@ -293,7 +285,6 @@ dependencies = [ "heapsize 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -664,16 +655,6 @@ dependencies = [ "libc 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rayon" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "regex" version = "0.1.54" diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 852ba6a36..858185873 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -138,9 +138,6 @@ pub trait BlockChainClient : Sync + Send { /// Get block total difficulty. fn block_total_difficulty(&self, id: BlockId) -> Option; - /// Get address nonce. - fn nonce(&self, address: &Address) -> U256; - /// Get block hash. fn block_hash(&self, id: BlockId) -> Option; @@ -368,14 +365,18 @@ impl Client where V: Verifier { bad_blocks.insert(header.hash()); continue; } + let closed_block = self.check_and_close_block(&block); if let Err(_) = closed_block { bad_blocks.insert(header.hash()); break; } + + // Insert block + let closed_block = closed_block.unwrap(); + self.chain.write().unwrap().insert_block(&block.bytes, closed_block.block().receipts().clone()); good_blocks.push(header.hash()); - // Are we committing an era? let ancient = if header.number() >= HISTORY { let n = header.number() - HISTORY; let chain = self.chain.read().unwrap(); @@ -385,16 +386,10 @@ impl Client where V: Verifier { }; // Commit results - let closed_block = closed_block.unwrap(); - let receipts = closed_block.block().receipts().clone(); closed_block.drain() .commit(header.number(), &header.hash(), ancient) .expect("State DB commit failed."); - // And update the chain - self.chain.write().unwrap() - .insert_block(&block.bytes, receipts); - self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); } @@ -413,7 +408,7 @@ impl Client where V: Verifier { if !good_blocks.is_empty() && block_queue.queue_info().is_empty() { io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { good: good_blocks, - retracted: bad_blocks, + bad: bad_blocks, })).unwrap(); } } @@ -586,10 +581,6 @@ impl BlockChainClient for Client where V: Verifier { Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) } - fn nonce(&self, address: &Address) -> U256 { - self.state().nonce(address) - } - fn block_hash(&self, id: BlockId) -> Option { let chain = self.chain.read().unwrap(); Self::block_hash(&chain, id) diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index a80adb0ba..756d02407 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -30,7 +30,7 @@ pub enum SyncMessage { /// Hashes of blocks imported to blockchain good: Vec, /// Hashes of blocks not imported to blockchain - retracted: Vec, + bad: Vec, }, /// A block is ready BlockVerified, diff --git a/sync/Cargo.toml b/sync/Cargo.toml index 0097cd47e..f10a772e3 100644 --- a/sync/Cargo.toml +++ b/sync/Cargo.toml @@ -17,7 +17,6 @@ time = "0.1.34" rand = "0.3.13" heapsize = "0.3" rustc-serialize = "0.3" -rayon = "0.3.1" [features] default = [] diff --git a/sync/src/chain.rs b/sync/src/chain.rs index ddf30854a..530cfa424 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -30,17 +30,14 @@ /// use util::*; -use rayon::prelude::*; use std::mem::{replace}; -use ethcore::views::{HeaderView, BlockView}; +use ethcore::views::{HeaderView}; use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo}; use range_collection::{RangeCollection, ToUsize, FromUsize}; use ethcore::error::*; use ethcore::block::Block; -use ethcore::transaction::SignedTransaction; use io::SyncIo; -use transaction_queue::TransactionQueue; use time; use super::SyncConfig; @@ -212,8 +209,6 @@ pub struct ChainSync { max_download_ahead_blocks: usize, /// Network ID network_id: U256, - /// Transactions Queue - transaction_queue: Mutex, } type RlpResponseResult = Result, PacketDecodeError>; @@ -239,7 +234,6 @@ impl ChainSync { last_send_block_number: 0, max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), network_id: config.network_id, - transaction_queue: Mutex::new(TransactionQueue::new()), } } @@ -298,7 +292,6 @@ impl ChainSync { self.starting_block = 0; self.highest_block = None; self.have_common_block = false; - self.transaction_queue.lock().unwrap().clear(); self.starting_block = io.chain().chain_info().best_block_number; self.state = SyncState::NotSynced; } @@ -491,7 +484,7 @@ impl ChainSync { trace!(target: "sync", "New block already queued {:?}", h); }, Ok(_) => { - if self.current_base_block() < header.number { + if self.current_base_block() < header.number { self.last_imported_block = Some(header.number); self.remove_downloaded_blocks(header.number); } @@ -928,16 +921,8 @@ impl ChainSync { } } /// Called when peer sends us new transactions - fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { - let chain = io.chain(); - let item_count = r.item_count(); - trace!(target: "sync", "{} -> Transactions ({} entries)", peer_id, item_count); - let fetch_latest_nonce = |a : &Address| chain.nonce(a); - for i in 0..item_count { - let tx: SignedTransaction = try!(r.val_at(i)); - self.transaction_queue.lock().unwrap().add(tx, &fetch_latest_nonce); - } - Ok(()) + fn on_peer_transactions(&mut self, _io: &mut SyncIo, _peer_id: PeerId, _r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + Ok(()) } /// Send Status message @@ -1263,37 +1248,6 @@ impl ChainSync { } self.last_send_block_number = chain.best_block_number; } - - /// called when block is imported to chain, updates transactions queue - pub fn chain_new_blocks(&mut self, io: &SyncIo, good: &[H256], retracted: &[H256]) { - fn fetch_transactions(chain: &BlockChainClient, hash: &H256) -> Vec { - let block = chain - .block(BlockId::Hash(hash.clone())) - // Client should send message after commit to db and inserting to chain. - .expect("Expected in-chain blocks."); - let block = BlockView::new(&block); - block.transactions() - } - - - let chain = io.chain(); - let good = good.par_iter().map(|h| fetch_transactions(chain, h)); - let retracted = retracted.par_iter().map(|h| fetch_transactions(chain, h)); - - good.for_each(|txs| { - let mut transaction_queue = self.transaction_queue.lock().unwrap(); - let hashes = txs.iter().map(|tx| tx.hash()).collect::>(); - transaction_queue.remove_all(&hashes, |a| chain.nonce(a)); - }); - retracted.for_each(|txs| { - // populate sender - for tx in &txs { - let _sender = tx.sender(); - } - let mut transaction_queue = self.transaction_queue.lock().unwrap(); - transaction_queue.add_all(txs, |a| chain.nonce(a)); - }); - } } #[cfg(test)] @@ -1434,7 +1388,7 @@ mod tests { #[test] fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10)); let chain_info = client.chain_info(); @@ -1448,7 +1402,7 @@ mod tests { #[test] fn calculates_tree_for_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(15, EachBlockWith::Uncle); + client.add_blocks(15, false); let start = client.block_hash_delta_minus(4); let end = client.block_hash_delta_minus(2); @@ -1465,7 +1419,7 @@ mod tests { #[test] fn sends_new_hashes_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1484,7 +1438,7 @@ mod tests { #[test] fn sends_latest_block_to_lagging_peer() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1502,7 +1456,7 @@ mod tests { #[test] fn handles_peer_new_block_mallformed() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); + client.add_blocks(10, false); let block_data = get_dummy_block(11, client.chain_info().best_block_hash); @@ -1520,7 +1474,7 @@ mod tests { #[test] fn handles_peer_new_block() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); + client.add_blocks(10, false); let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); @@ -1538,7 +1492,7 @@ mod tests { #[test] fn handles_peer_new_block_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); + client.add_blocks(10, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1554,7 +1508,7 @@ mod tests { #[test] fn handles_peer_new_hashes() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); + client.add_blocks(10, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1570,7 +1524,7 @@ mod tests { #[test] fn handles_peer_new_hashes_empty() { let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); + client.add_blocks(10, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let mut io = TestIo::new(&mut client, &mut queue, None); @@ -1588,7 +1542,7 @@ mod tests { #[test] fn hashes_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1606,7 +1560,7 @@ mod tests { #[test] fn block_rlp_mutually_acceptable() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); let chain_info = client.chain_info(); @@ -1619,37 +1573,10 @@ mod tests { assert!(result.is_ok()); } - #[test] - fn should_add_transactions_to_queue() { - // given - let mut client = TestBlockChainClient::new(); - client.add_blocks(98, EachBlockWith::Uncle); - client.add_blocks(1, EachBlockWith::UncleAndTransaction); - client.add_blocks(1, EachBlockWith::Transaction); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5)); - - let good_blocks = vec![client.block_hash_delta_minus(2)]; - let retracted_blocks = vec![client.block_hash_delta_minus(1)]; - - let mut queue = VecDeque::new(); - let io = TestIo::new(&mut client, &mut queue, None); - - // when - sync.chain_new_blocks(&io, &[], &good_blocks); - assert_eq!(sync.transaction_queue.lock().unwrap().status().future, 0); - assert_eq!(sync.transaction_queue.lock().unwrap().status().pending, 1); - sync.chain_new_blocks(&io, &good_blocks, &retracted_blocks); - - // then - let status = sync.transaction_queue.lock().unwrap().status(); - assert_eq!(status.pending, 1); - assert_eq!(status.future, 0); - } - #[test] fn returns_requested_block_headers() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); @@ -1673,7 +1600,7 @@ mod tests { #[test] fn returns_requested_block_headers_reverse() { let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); + client.add_blocks(100, false); let mut queue = VecDeque::new(); let io = TestIo::new(&mut client, &mut queue, None); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index d67a09f3b..74541660d 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -54,7 +54,6 @@ extern crate ethcore; extern crate env_logger; extern crate time; extern crate rand; -extern crate rayon; #[macro_use] extern crate heapsize; @@ -71,7 +70,8 @@ use io::NetSyncIo; mod chain; mod io; mod range_collection; -mod transaction_queue; +// TODO [todr] Made public to suppress dead code warnings +pub mod transaction_queue; #[cfg(test)] mod tests; @@ -153,14 +153,8 @@ impl NetworkProtocolHandler for EthSync { } fn message(&self, io: &NetworkContext, message: &SyncMessage) { - match *message { - SyncMessage::BlockVerified => { - self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); - }, - SyncMessage::NewChainBlocks { ref good, ref retracted } => { - let sync_io = NetSyncIo::new(io, self.chain.deref()); - self.sync.write().unwrap().chain_new_blocks(&sync_io, good, retracted); - } + if let SyncMessage::BlockVerified = *message { + self.sync.write().unwrap().chain_blocks_verified(&mut NetSyncIo::new(io, self.chain.deref())); } } } diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index 58f50916e..b01c894a0 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -24,8 +24,8 @@ use super::helpers::*; fn two_peers() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, false); + net.peer_mut(2).chain.add_blocks(1000, false); net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); assert_eq!(net.peer(0).chain.blocks.read().unwrap().deref(), net.peer(1).chain.blocks.read().unwrap().deref()); @@ -35,8 +35,8 @@ fn two_peers() { fn status_after_sync() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, false); + net.peer_mut(2).chain.add_blocks(1000, false); net.sync(); let status = net.peer(0).sync.status(); assert_eq!(status.state, SyncState::Idle); @@ -45,8 +45,8 @@ fn status_after_sync() { #[test] fn takes_few_steps() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(100, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(100, false); + net.peer_mut(2).chain.add_blocks(100, false); let total_steps = net.sync(); assert!(total_steps < 7); } @@ -56,9 +56,8 @@ fn empty_blocks() { ::env_logger::init().ok(); let mut net = TestNet::new(3); for n in 0..200 { - let with = if n % 2 == 0 { EachBlockWith::Nothing } else { EachBlockWith::Uncle }; - net.peer_mut(1).chain.add_blocks(5, with.clone()); - net.peer_mut(2).chain.add_blocks(5, with); + net.peer_mut(1).chain.add_blocks(5, n % 2 == 0); + net.peer_mut(2).chain.add_blocks(5, n % 2 == 0); } net.sync(); assert!(net.peer(0).chain.block(BlockId::Number(1000)).is_some()); @@ -69,14 +68,14 @@ fn empty_blocks() { fn forked() { ::env_logger::init().ok(); let mut net = TestNet::new(3); - net.peer_mut(0).chain.add_blocks(300, EachBlockWith::Uncle); - net.peer_mut(1).chain.add_blocks(300, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(300, EachBlockWith::Uncle); - net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Nothing); //fork - net.peer_mut(1).chain.add_blocks(200, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(200, EachBlockWith::Uncle); - net.peer_mut(1).chain.add_blocks(100, EachBlockWith::Uncle); //fork between 1 and 2 - net.peer_mut(2).chain.add_blocks(10, EachBlockWith::Nothing); + net.peer_mut(0).chain.add_blocks(300, false); + net.peer_mut(1).chain.add_blocks(300, false); + net.peer_mut(2).chain.add_blocks(300, false); + net.peer_mut(0).chain.add_blocks(100, true); //fork + net.peer_mut(1).chain.add_blocks(200, false); + net.peer_mut(2).chain.add_blocks(200, false); + net.peer_mut(1).chain.add_blocks(100, false); //fork between 1 and 2 + net.peer_mut(2).chain.add_blocks(10, true); // peer 1 has the best chain of 601 blocks let peer1_chain = net.peer(1).chain.numbers.read().unwrap().clone(); net.sync(); @@ -88,8 +87,8 @@ fn forked() { #[test] fn restart() { let mut net = TestNet::new(3); - net.peer_mut(1).chain.add_blocks(1000, EachBlockWith::Uncle); - net.peer_mut(2).chain.add_blocks(1000, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(1000, false); + net.peer_mut(2).chain.add_blocks(1000, false); net.sync_steps(8); @@ -110,8 +109,8 @@ fn status_empty() { #[test] fn status_packet() { let mut net = TestNet::new(2); - net.peer_mut(0).chain.add_blocks(100, EachBlockWith::Uncle); - net.peer_mut(1).chain.add_blocks(1, EachBlockWith::Uncle); + net.peer_mut(0).chain.add_blocks(100, false); + net.peer_mut(1).chain.add_blocks(1, false); net.start(); @@ -124,10 +123,10 @@ fn status_packet() { #[test] fn propagate_hashes() { let mut net = TestNet::new(6); - net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, false); net.sync(); - net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, false); net.sync(); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -150,10 +149,10 @@ fn propagate_hashes() { #[test] fn propagate_blocks() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, false); net.sync(); - net.peer_mut(0).chain.add_blocks(10, EachBlockWith::Uncle); + net.peer_mut(0).chain.add_blocks(10, false); net.trigger_block_verified(0); //first event just sets the marker net.trigger_block_verified(0); @@ -165,7 +164,7 @@ fn propagate_blocks() { #[test] fn restart_on_malformed_block() { let mut net = TestNet::new(2); - net.peer_mut(1).chain.add_blocks(10, EachBlockWith::Uncle); + net.peer_mut(1).chain.add_blocks(10, false); net.peer_mut(1).chain.corrupt_block(6); net.sync_steps(10); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 5b53ad90b..e170a4a85 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -22,7 +22,7 @@ use io::SyncIo; use chain::ChainSync; use ::SyncConfig; use ethcore::receipt::Receipt; -use ethcore::transaction::{LocalizedTransaction, Transaction, Action}; +use ethcore::transaction::LocalizedTransaction; use ethcore::filter::Filter; use ethcore::log_entry::LocalizedLogEntry; @@ -34,14 +34,6 @@ pub struct TestBlockChainClient { pub difficulty: RwLock, } -#[derive(Clone)] -pub enum EachBlockWith { - Nothing, - Uncle, - Transaction, - UncleAndTransaction -} - impl TestBlockChainClient { pub fn new() -> TestBlockChainClient { @@ -52,53 +44,30 @@ impl TestBlockChainClient { last_hash: RwLock::new(H256::new()), difficulty: RwLock::new(From::from(0)), }; - client.add_blocks(1, EachBlockWith::Nothing); // add genesis block + client.add_blocks(1, true); // add genesis block client.genesis_hash = client.last_hash.read().unwrap().clone(); client } - pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { + pub fn add_blocks(&mut self, count: usize, empty: bool) { let len = self.numbers.read().unwrap().len(); for n in len..(len + count) { let mut header = BlockHeader::new(); header.difficulty = From::from(n); header.parent_hash = self.last_hash.read().unwrap().clone(); header.number = n as BlockNumber; - let uncles = match with { - EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => { - let mut uncles = RlpStream::new_list(1); - let mut uncle_header = BlockHeader::new(); - uncle_header.difficulty = From::from(n); - uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); - uncle_header.number = n as BlockNumber; - uncles.append(&uncle_header); - header.uncles_hash = uncles.as_raw().sha3(); - uncles - }, - _ => RlpStream::new_list(0) - }; - let txs = match with { - EachBlockWith::Transaction | EachBlockWith::UncleAndTransaction => { - let mut txs = RlpStream::new_list(1); - let keypair = KeyPair::create().unwrap(); - let tx = Transaction { - action: Action::Create, - value: U256::from(100), - data: "3331600055".from_hex().unwrap(), - gas: U256::from(100_000), - gas_price: U256::one(), - nonce: U256::zero() - }; - let signed_tx = tx.sign(&keypair.secret()); - txs.append(&signed_tx); - txs.out() - }, - _ => rlp::NULL_RLP.to_vec() - }; - + let mut uncles = RlpStream::new_list(if empty {0} else {1}); + if !empty { + let mut uncle_header = BlockHeader::new(); + uncle_header.difficulty = From::from(n); + uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); + uncle_header.number = n as BlockNumber; + uncles.append(&uncle_header); + header.uncles_hash = uncles.as_raw().sha3(); + } let mut rlp = RlpStream::new_list(3); rlp.append(&header); - rlp.append_raw(&txs, 1); + rlp.append_raw(&rlp::NULL_RLP, 1); rlp.append_raw(uncles.as_raw(), 1); self.import_block(rlp.as_raw().to_vec()).unwrap(); } @@ -140,10 +109,6 @@ impl BlockChainClient for TestBlockChainClient { unimplemented!(); } - fn nonce(&self, _address: &Address) -> U256 { - U256::zero() - } - fn code(&self, _address: &Address) -> Option { unimplemented!(); } diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 83665dfda..4f5622a2f 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -219,19 +219,19 @@ impl TransactionQueue { /// Removes all transactions identified by hashes given in slice /// /// If gap is introduced marks subsequent transactions as future - pub fn remove_all(&mut self, transaction_hashes: &[H256], fetch_nonce: T) + pub fn remove_all(&mut self, txs: &[H256], fetch_nonce: T) where T: Fn(&Address) -> U256 { - for hash in transaction_hashes { - self.remove(&hash, &fetch_nonce); + for tx in txs { + self.remove(&tx, &fetch_nonce); } } /// Removes transaction identified by hashes from queue. /// /// If gap is introduced marks subsequent transactions as future - pub fn remove(&mut self, transaction_hash: &H256, fetch_nonce: &T) + pub fn remove(&mut self, hash: &H256, fetch_nonce: &T) where T: Fn(&Address) -> U256 { - let transaction = self.by_hash.remove(transaction_hash); + let transaction = self.by_hash.remove(hash); if transaction.is_none() { // We don't know this transaction return; @@ -240,6 +240,7 @@ impl TransactionQueue { let sender = transaction.sender(); let nonce = transaction.nonce(); + println!("Removing tx: {:?}", transaction.transaction); // Remove from future self.future.drop(&sender, &nonce); @@ -265,6 +266,7 @@ impl TransactionQueue { // Goes to future or is removed let order = self.current.drop(&sender, &k).unwrap(); if k >= current_nonce { + println!("Moving to future: {:?}", order); self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); } else { self.by_hash.remove(&order.hash); @@ -274,7 +276,7 @@ impl TransactionQueue { // And now lets check if there is some chain of transactions in future // that should be placed in current - if let Some(new_current_top) = self.move_future_txs(sender.clone(), current_nonce, current_nonce) { + if let Some(new_current_top) = self.move_future_txs(sender.clone(), current_nonce - U256::one(), current_nonce) { self.last_nonces.insert(sender, new_current_top); } } @@ -297,7 +299,9 @@ impl TransactionQueue { self.last_nonces.clear(); } - fn move_future_txs(&mut self, address: Address, mut current_nonce: U256, first_nonce: U256) -> Option { + fn move_future_txs(&mut self, address: Address, current_nonce: U256, first_nonce: U256) -> Option { + println!("Moving from future for: {:?} base: {:?}", current_nonce, first_nonce); + let mut current_nonce = current_nonce + U256::one(); { let by_nonce = self.future.by_address.row_mut(&address); if let None = by_nonce { @@ -308,6 +312,7 @@ impl TransactionQueue { // remove also from priority and hash self.future.by_priority.remove(&order); // Put to current + println!("Moved: {:?}", order); let order = order.update_height(current_nonce.clone(), first_nonce); self.current.insert(address.clone(), current_nonce, order); current_nonce = current_nonce + U256::one(); @@ -328,6 +333,7 @@ impl TransactionQueue { .cloned() .map_or_else(|| fetch_nonce(&address), |n| n + U256::one()); + println!("Expected next: {:?}, got: {:?}", next_nonce, nonce); // Check height if nonce > next_nonce { let order = TransactionOrder::for_transaction(&tx, next_nonce); @@ -339,7 +345,6 @@ impl TransactionQueue { return; } else if next_nonce > nonce { // Droping transaction - trace!(target: "sync", "Dropping transaction with nonce: {} - expecting: {}", nonce, next_nonce); return; } @@ -351,7 +356,7 @@ impl TransactionQueue { // Insert to current self.current.insert(address.clone(), nonce, order); // But maybe there are some more items waiting in future? - let new_last_nonce = self.move_future_txs(address.clone(), nonce + U256::one(), base_nonce); + let new_last_nonce = self.move_future_txs(address.clone(), nonce, base_nonce); self.last_nonces.insert(address.clone(), new_last_nonce.unwrap_or(nonce)); // Enforce limit self.current.enforce_limit(&self.by_hash); From 003d1fd0cc1c1f815c8f0b772baefd374dab67aa Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 5 Mar 2016 23:09:51 +0100 Subject: [PATCH 89/89] Network tracing improvements --- sync/src/chain.rs | 2 +- util/src/network/connection.rs | 10 ++++---- util/src/network/handshake.rs | 14 +++++------ util/src/network/host.rs | 43 +++++++++++++++++++--------------- util/src/network/session.rs | 9 ++++--- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 530cfa424..63640f87f 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -575,7 +575,7 @@ impl ChainSync { pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) { trace!(target: "sync", "== Connected {}", peer); if let Err(e) = self.send_status(io) { - warn!(target:"sync", "Error sending status request: {:?}", e); + trace!(target:"sync", "Error sending status request: {:?}", e); io.disable_peer(peer); } } diff --git a/util/src/network/connection.rs b/util/src/network/connection.rs index 55e688c91..fe65be6d1 100644 --- a/util/src/network/connection.rs +++ b/util/src/network/connection.rs @@ -190,25 +190,25 @@ impl Connection { /// Register this connection with the IO event loop. pub fn register_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection register; token={:?}", reg); + trace!(target: "network", "connection register; token={:?}", reg); if let Err(e) = event_loop.register(&self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()) { - debug!("Failed to register {:?}, {:?}", reg, e); + trace!(target: "network", "Failed to register {:?}, {:?}", reg, e); } Ok(()) } /// Update connection registration. Should be called at the end of the IO handler. pub fn update_socket(&self, reg: Token, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection reregister; token={:?}", reg); + trace!(target: "network", "connection reregister; token={:?}", reg); event_loop.reregister( &self.socket, reg, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - debug!("Failed to reregister {:?}, {:?}", reg, e); + trace!(target: "network", "Failed to reregister {:?}, {:?}", reg, e); Ok(()) }) } /// Delete connection registration. Should be called at the end of the IO handler. pub fn deregister_socket(&self, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection deregister; token={:?}", self.token); + trace!(target: "network", "connection deregister; token={:?}", self.token); event_loop.deregister(&self.socket).ok(); // ignore errors here Ok(()) } diff --git a/util/src/network/handshake.rs b/util/src/network/handshake.rs index cca133ba9..a72cc28ad 100644 --- a/util/src/network/handshake.rs +++ b/util/src/network/handshake.rs @@ -222,7 +222,7 @@ impl Handshake { /// Parse, validate and confirm auth message fn read_auth(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received handshake auth from {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Received handshake auth from {:?}", self.connection.socket.peer_addr()); if data.len() != V4_AUTH_PACKET_SIZE { debug!(target:"net", "Wrong auth packet size"); return Err(From::from(NetworkError::BadProtocol)); @@ -253,7 +253,7 @@ impl Handshake { } fn read_auth_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); self.auth_cipher.extend_from_slice(data); let auth = try!(ecies::decrypt(secret, &self.auth_cipher[0..2], &self.auth_cipher[2..])); let rlp = UntrustedRlp::new(&auth); @@ -268,7 +268,7 @@ impl Handshake { /// Parse and validate ack message fn read_ack(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); if data.len() != V4_ACK_PACKET_SIZE { debug!(target:"net", "Wrong ack packet size"); return Err(From::from(NetworkError::BadProtocol)); @@ -296,7 +296,7 @@ impl Handshake { } fn read_ack_eip8(&mut self, secret: &Secret, data: &[u8]) -> Result<(), UtilError> { - trace!(target:"net", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Received EIP8 handshake auth from {:?}", self.connection.socket.peer_addr()); self.ack_cipher.extend_from_slice(data); let ack = try!(ecies::decrypt(secret, &self.ack_cipher[0..2], &self.ack_cipher[2..])); let rlp = UntrustedRlp::new(&ack); @@ -309,7 +309,7 @@ impl Handshake { /// Sends auth message fn write_auth(&mut self, secret: &Secret, public: &Public) -> Result<(), UtilError> { - trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants let len = data.len(); { @@ -336,7 +336,7 @@ impl Handshake { /// Sends ack message fn write_ack(&mut self) -> Result<(), UtilError> { - trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Sending handshake ack to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants let len = data.len(); { @@ -355,7 +355,7 @@ impl Handshake { /// Sends EIP8 ack message fn write_ack_eip8(&mut self) -> Result<(), UtilError> { - trace!(target:"net", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr()); + trace!(target:"network", "Sending EIP8 handshake ack to {:?}", self.connection.socket.peer_addr()); let mut rlp = RlpStream::new_list(3); rlp.append(self.ecdhe.public()); rlp.append(&self.nonce); diff --git a/util/src/network/host.rs b/util/src/network/host.rs index f2cc9fe48..ece24a1d1 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -170,29 +170,37 @@ pub struct NetworkContext<'s, Message> where Message: Send + Sync + Clone + 'sta io: &'s IoContext>, protocol: ProtocolId, sessions: Arc>>, - session: Option, + session: Option, + session_id: Option, } impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone + 'static, { /// Create a new network IO access point. Takes references to all the data that can be updated within the IO handler. fn new(io: &'s IoContext>, protocol: ProtocolId, - session: Option, sessions: Arc>>) -> NetworkContext<'s, Message> { + session: Option, sessions: Arc>>) -> NetworkContext<'s, Message> { + let id = session.as_ref().map(|s| s.lock().unwrap().token()); NetworkContext { io: io, protocol: protocol, + session_id: id, session: session, sessions: sessions, } } + fn resolve_session(&self, peer: PeerId) -> Option { + match self.session_id { + Some(id) if id == peer => self.session.clone(), + _ => self.sessions.read().unwrap().get(peer).cloned(), + } + } + /// Send a packet over the network to another peer. pub fn send(&self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { - let session = { self.sessions.read().unwrap().get(peer).cloned() }; + let session = self.resolve_session(peer); if let Some(session) = session { - session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "network", "Send error: {:?}", e); - }); //TODO: don't copy vector data + try!(session.lock().unwrap().deref_mut().send_packet(self.protocol, packet_id as u8, &data)); try!(self.io.update_registration(peer)); } else { trace!(target: "network", "Send: Peer no longer exist") @@ -200,14 +208,10 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone Ok(()) } - /// Respond to a current network message. Panics if no there is no packet in the context. + /// Respond to a current network message. Panics if no there is no packet in the context. If the session is expired returns nothing. pub fn respond(&self, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { - match self.session { - Some(session) => self.send(session, packet_id, data), - None => { - panic!("Respond: Session does not exist") - } - } + assert!(self.session.is_some(), "Respond called without network context"); + self.send(self.session_id.unwrap(), packet_id, data) } /// Send an IO message @@ -215,7 +219,6 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone self.io.message(NetworkIoMessage::User(msg)); } - /// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected. pub fn disable_peer(&self, peer: PeerId) { //TODO: remove capability, disconnect if no capabilities left @@ -239,7 +242,7 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + Sync + Clone /// Returns peer identification string pub fn peer_info(&self, peer: PeerId) -> String { - let session = { self.sessions.read().unwrap().get(peer).cloned() }; + let session = self.resolve_session(peer); if let Some(session) = session { return session.lock().unwrap().info.client_version.clone() } @@ -624,7 +627,7 @@ impl Host where Message: Send + Sync + Clone { let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; let mut kill = false; let session = { self.sessions.read().unwrap().get(token).cloned() }; - if let Some(session) = session { + if let Some(session) = session.clone() { let mut s = session.lock().unwrap(); match s.readable(io, &self.info.read().unwrap()) { Err(e) => { @@ -656,11 +659,11 @@ impl Host where Message: Send + Sync + Clone { } for p in ready_data { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.connected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token); + h.connected(&NetworkContext::new(io, p, session.clone(), self.sessions.clone()), &token); } if let Some((p, packet_id, data)) = packet_data { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.read(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token, packet_id, &data[1..]); + h.read(&NetworkContext::new(io, p, session.clone(), self.sessions.clone()), &token, packet_id, &data[1..]); } io.update_registration(token).unwrap_or_else(|e| debug!(target: "network", "Token registration error: {:?}", e)); } @@ -718,6 +721,7 @@ impl Host where Message: Send + Sync + Clone { let mut to_disconnect: Vec = Vec::new(); let mut failure_id = None; let mut deregister = false; + let mut expired_session = None; match token { FIRST_HANDSHAKE ... LAST_HANDSHAKE => { let handshakes = self.handshakes.write().unwrap(); @@ -733,6 +737,7 @@ impl Host where Message: Send + Sync + Clone { FIRST_SESSION ... LAST_SESSION => { let sessions = self.sessions.write().unwrap(); if let Some(session) = sessions.get(token).cloned() { + expired_session = Some(session.clone()); let mut s = session.lock().unwrap(); if !s.expired() { if s.is_ready() { @@ -757,7 +762,7 @@ impl Host where Message: Send + Sync + Clone { } for p in to_disconnect { let h = self.handlers.read().unwrap().get(p).unwrap().clone(); - h.disconnected(&NetworkContext::new(io, p, Some(token), self.sessions.clone()), &token); + h.disconnected(&NetworkContext::new(io, p, expired_session.clone(), self.sessions.clone()), &token); } if deregister { io.deregister_stream(token).expect("Error deregistering stream"); diff --git a/util/src/network/session.rs b/util/src/network/session.rs index edf929a9a..84c063c92 100644 --- a/util/src/network/session.rs +++ b/util/src/network/session.rs @@ -213,6 +213,9 @@ impl Session { /// Send a protocol packet to peer. pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), UtilError> { + if self.expired() { + return Err(From::from(NetworkError::Expired)); + } let mut i = 0usize; while protocol != self.info.capabilities[i].protocol { i += 1; @@ -351,15 +354,15 @@ impl Session { offset += caps[i].packet_count; i += 1; } - trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); + trace!(target: "network", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); self.info.client_version = client_version; self.info.capabilities = caps; if self.info.capabilities.is_empty() { - trace!("No common capabilities with peer."); + trace!(target: "network", "No common capabilities with peer."); return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); } if protocol != host.protocol_version { - trace!("Peer protocol version mismatch: {}", protocol); + trace!(target: "network", "Peer protocol version mismatch: {}", protocol); return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); } self.had_hello = true;