Fix deadlock in blockchain. (#8977)

* snap: downgrade rust to revision 1.26.2, ref snapcraft/+bug/1778530

* snap: use plugin rust

.gitlab-ci.yml

@@ -180,15 +180,35 @@ windows:
     paths:
     - parity.zip
     name: "x86_64-pc-windows-msvc_parity"
+android-armv7:
+  stage: build
+  image: parity/parity-android:latest
+  only:
+    - beta
+    - tags
+    - stable
+    - triggers
+  script:
+    - cargo build --target=armv7-linux-androideabi
+  tags:
+    - rust-arm
+  allow_failure: true
+  artifacts:
+    paths:
+    - parity.zip
+    name: "armv7-linux-androideabi_parity"
 docker-build:
   stage: build
   only:
     - tags
+    - master
+    - beta
+    - stable
     - triggers
   before_script:
     - docker info
   script:
-    - if [ "$CI_BUILD_REF_NAME" == "beta-release" ]; then DOCKER_TAG="latest"; else DOCKER_TAG=$CI_BUILD_REF_NAME; fi
+    - if [ "$CI_BUILD_REF_NAME" == "master" ]; then DOCKER_TAG="latest"; else DOCKER_TAG=$CI_BUILD_REF_NAME; fi
     - echo "Tag:" $DOCKER_TAG
     - docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity
     - scripts/docker-build.sh $DOCKER_TAG

Cargo.toml

@@ -2,7 +2,7 @@
 description = "Parity Ethereum client"
 name = "parity"
 # NOTE Make sure to update util/version/Cargo.toml as well
-version = "1.11.0"
+version = "1.11.5"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 

ethcore/light/src/client/header_chain.rs

@@ -228,7 +228,7 @@ impl HeaderChain {
 		let decoded_header = spec.genesis_header();
 
 		let chain = if let Some(current) = db.get(col, CURRENT_KEY)? {
-			let curr : BestAndLatest = ::rlp::decode(&current);
+			let curr : BestAndLatest = ::rlp::decode(&current).expect("decoding db value failed");
 
 			let mut cur_number = curr.latest_num;
 			let mut candidates = BTreeMap::new();
@@ -236,7 +236,7 @@ impl HeaderChain {
 			// load all era entries, referenced headers within them,
 			// and live epoch proofs.
 			while let Some(entry) = db.get(col, era_key(cur_number).as_bytes())? {
-				let entry: Entry = ::rlp::decode(&entry);
+				let entry: Entry = ::rlp::decode(&entry).expect("decoding db value failed");
 				trace!(target: "chain", "loaded header chain entry for era {} with {} candidates",
 					cur_number, entry.candidates.len());
 
@@ -305,7 +305,7 @@ impl HeaderChain {
 					batch.put(col, cht_key(cht_num as u64).as_bytes(), &::rlp::encode(cht_root));
 				}
 
-				let decoded_header = hardcoded_sync.header.decode();
+				let decoded_header = hardcoded_sync.header.decode()?;
 				let decoded_header_num = decoded_header.number();
 
 				// write the block in the DB.
@@ -524,7 +524,10 @@ impl HeaderChain {
 											None
 										}
 										Ok(None) => panic!("stored candidates always have corresponding headers; qed"),
-										Ok(Some(header)) => Some((epoch_transition, ::rlp::decode(&header))),
+										Ok(Some(header)) => Some((
+											epoch_transition,
+											::rlp::decode(&header).expect("decoding value from db failed")
+										)),
 									};
 								}
 							}
@@ -582,7 +585,7 @@ impl HeaderChain {
 						bail!(ErrorKind::Database(msg.into()));
 					};
 
-					let decoded = header.decode();
+					let decoded = header.decode().expect("decoding db value failed");
 
 					let entry: Entry = {
 						let bytes = self.db.get(self.col, era_key(h_num).as_bytes())?
@@ -591,7 +594,7 @@ impl HeaderChain {
 													in an inconsistent state", h_num);
 								ErrorKind::Database(msg.into())
 							})?;
-						::rlp::decode(&bytes)
+						::rlp::decode(&bytes).expect("decoding db value failed")
 					};
 
 					let total_difficulty = entry.candidates.iter()
@@ -604,9 +607,9 @@ impl HeaderChain {
 						.total_difficulty;
 
 					break Ok(Some(SpecHardcodedSync {
-						header: header,
-						total_difficulty: total_difficulty,
-						chts: chts,
+						header,
+						total_difficulty,
+						chts,
 					}));
 				},
 				None => {
@@ -742,7 +745,7 @@ impl HeaderChain {
 	/// so including it within a CHT would be redundant.
 	pub fn cht_root(&self, n: usize) -> Option<H256> {
 		match self.db.get(self.col, cht_key(n as u64).as_bytes()) {
-			Ok(val) => val.map(|x| ::rlp::decode(&x)),
+			Ok(db_fetch) => db_fetch.map(|bytes| ::rlp::decode(&bytes).expect("decoding value from db failed")),
 			Err(e) => {
 				warn!(target: "chain", "Error reading from database: {}", e);
 				None
@@ -793,7 +796,7 @@ impl HeaderChain {
 	pub fn pending_transition(&self, hash: H256) -> Option<PendingEpochTransition> {
 		let key = pending_transition_key(hash);
 		match self.db.get(self.col, &*key) {
-			Ok(val) => val.map(|x| ::rlp::decode(&x)),
+			Ok(db_fetch) => db_fetch.map(|bytes| ::rlp::decode(&bytes).expect("decoding value from db failed")),
 			Err(e) => {
 				warn!(target: "chain", "Error reading from database: {}", e);
 				None
@@ -812,7 +815,9 @@ impl HeaderChain {
 
 		for hdr in self.ancestry_iter(BlockId::Hash(parent_hash)) {
 			if let Some(transition) = live_proofs.get(&hdr.hash()).cloned() {
-				return Some((hdr.decode(), transition.proof))
+				return hdr.decode().map(|decoded_hdr| {
+					(decoded_hdr, transition.proof)
+				}).ok();
 			}
 		}
 
@@ -1192,7 +1197,7 @@ mod tests {
 
 		let cache = Arc::new(Mutex::new(Cache::new(Default::default(), Duration::from_secs(6 * 3600))));
 
-		let chain = HeaderChain::new(db.clone(), None, &spec, cache, HardcodedSync::Allow).unwrap();
+		let chain = HeaderChain::new(db.clone(), None, &spec, cache, HardcodedSync::Allow).expect("failed to instantiate a new HeaderChain");
 
 		let mut parent_hash = genesis_header.hash();
 		let mut rolling_timestamp = genesis_header.timestamp();
@@ -1211,17 +1216,17 @@ mod tests {
 			parent_hash = header.hash();
 
 			let mut tx = db.transaction();
-			let pending = chain.insert(&mut tx, header, None).unwrap();
+			let pending = chain.insert(&mut tx, header, None).expect("failed inserting a transaction");
 			db.write(tx).unwrap();
 			chain.apply_pending(pending);
 
 			rolling_timestamp += 10;
 		}
 
-		let hardcoded_sync = chain.read_hardcoded_sync().unwrap().unwrap();
+		let hardcoded_sync = chain.read_hardcoded_sync().expect("failed reading hardcoded sync").expect("failed unwrapping hardcoded sync");
 		assert_eq!(hardcoded_sync.chts.len(), 3);
 		assert_eq!(hardcoded_sync.total_difficulty, total_difficulty);
-		let decoded: Header = hardcoded_sync.header.decode();
+		let decoded: Header = hardcoded_sync.header.decode().expect("decoding failed");
 		assert_eq!(decoded.number(), h_num);
 	}
 }

ethcore/light/src/client/mod.rs

@@ -318,7 +318,7 @@ impl<T: ChainDataFetcher> Client<T> {
 
 			let epoch_proof =  self.engine.is_epoch_end(
 				&verified_header,
-				&|h| self.chain.block_header(BlockId::Hash(h)).map(|hdr| hdr.decode()),
+				&|h| self.chain.block_header(BlockId::Hash(h)).and_then(|hdr| hdr.decode().ok()),
 				&|h| self.chain.pending_transition(h),
 			);
 
@@ -426,7 +426,15 @@ impl<T: ChainDataFetcher> Client<T> {
 		};
 
 		// Verify Block Family
-		let verify_family_result = self.engine.verify_block_family(&verified_header, &parent_header.decode());
+
+		let verify_family_result = {
+			parent_header.decode()
+				.map_err(|dec_err| dec_err.into())
+				.and_then(|decoded| {
+					self.engine.verify_block_family(&verified_header, &decoded)
+				})
+
+		};
 		if let Err(e) = verify_family_result {
 			warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}",
 				verified_header.number(), verified_header.hash(), e);

ethcore/light/src/client/service.rs

@@ -30,7 +30,7 @@ use kvdb::KeyValueDB;
 use cache::Cache;
 use parking_lot::Mutex;
 
-use super::{ChainDataFetcher, Client, Config as ClientConfig};
+use super::{ChainDataFetcher, LightChainNotify, Client, Config as ClientConfig};
 
 /// Errors on service initialization.
 #[derive(Debug)]
@@ -86,6 +86,11 @@ impl<T: ChainDataFetcher> Service<T> {
 		})
 	}
 
+	/// Set the actor to be notified on certain chain events
+	pub fn add_notify(&self, notify: Arc<LightChainNotify>) {
+		self.client.add_listener(Arc::downgrade(&notify));
+	}
+
 	/// Register an I/O handler on the service.
 	pub fn register_handler(&self, handler: Arc<IoHandler<ClientIoMessage> + Send>) -> Result<(), IoError> {
 		self.io_service.register_handler(handler)

ethcore/light/src/net/mod.rs

@@ -75,14 +75,17 @@ const RECALCULATE_COSTS_INTERVAL: Duration = Duration::from_secs(60 * 60);
 // minimum interval between updates.
 const UPDATE_INTERVAL: Duration = Duration::from_millis(5000);
 
+/// Packet count for PIP.
+const PACKET_COUNT_V1: u8 = 9;
+
 /// Supported protocol versions.
-pub const PROTOCOL_VERSIONS: &'static [u8] = &[1];
+pub const PROTOCOL_VERSIONS: &'static [(u8, u8)] = &[
+	(1, PACKET_COUNT_V1),
+];
 
 /// Max protocol version.
 pub const MAX_PROTOCOL_VERSION: u8 = 1;
 
-/// Packet count for PIP.
-pub const PACKET_COUNT: u8 = 9;
 
 // packet ID definitions.
 mod packet {
@@ -111,9 +114,9 @@ mod packet {
 mod timeout {
 	use std::time::Duration;
 
-	pub const HANDSHAKE: Duration =  Duration::from_millis(2500);
-	pub const ACKNOWLEDGE_UPDATE: Duration = Duration::from_millis(5000);
-	pub const BASE: u64 = 1500; // base timeout for packet.
+	pub const HANDSHAKE: Duration = Duration::from_millis(4_000);
+	pub const ACKNOWLEDGE_UPDATE: Duration = Duration::from_millis(5_000);
+	pub const BASE: u64 = 2_500; // base timeout for packet.
 
 	// timeouts per request within packet.
 	pub const HEADERS: u64 = 250; // per header?
@@ -688,7 +691,7 @@ impl LightProtocol {
 			Err(e) => { punish(*peer, io, e); return }
 		};
 
-		if PROTOCOL_VERSIONS.iter().find(|x| **x == proto_version).is_none() {
+		if PROTOCOL_VERSIONS.iter().find(|x| x.0 == proto_version).is_none() {
 			punish(*peer, io, Error::UnsupportedProtocolVersion(proto_version));
 			return;
 		}

ethcore/light/src/net/request_credits.rs

@@ -407,7 +407,7 @@ mod tests {
 		let costs = CostTable::default();
 		let serialized = ::rlp::encode(&costs);
 
-		let new_costs: CostTable = ::rlp::decode(&*serialized);
+		let new_costs: CostTable = ::rlp::decode(&*serialized).unwrap();
 
 		assert_eq!(costs, new_costs);
 	}

ethcore/light/src/provider.rs

@@ -54,6 +54,7 @@ pub trait Provider: Send + Sync {
 	/// results within must adhere to the `skip` and `reverse` parameters.
 	fn block_headers(&self, req: request::CompleteHeadersRequest) -> Option<request::HeadersResponse> {
 		use request::HashOrNumber;
+		const MAX_HEADERS_TO_SEND: u64 = 512;
 
 		if req.max == 0 { return None }
 
@@ -82,10 +83,12 @@ pub trait Provider: Send + Sync {
 			}
 		};
 
-		let headers: Vec<_> = (0u64..req.max as u64)
-			.map(|x: u64| x.saturating_mul(req.skip + 1))
+		let max = ::std::cmp::min(MAX_HEADERS_TO_SEND, req.max);
+
+		let headers: Vec<_> = (0u64..max)
+			.map(|x: u64| x.saturating_mul(req.skip.saturating_add(1)))
 			.take_while(|x| if req.reverse { x < &start_num } else { best_num.saturating_sub(start_num) >= *x })
-			.map(|x| if req.reverse { start_num - x } else { start_num + x })
+			.map(|x| if req.reverse { start_num.saturating_sub(x) } else { start_num.saturating_add(x) })
 			.map(|x| self.block_header(BlockId::Number(x)))
 			.take_while(|x| x.is_some())
 			.flat_map(|x| x)

ethcore/light/src/types/request/mod.rs

@@ -1642,7 +1642,7 @@ mod tests {
 	{
 		// check as single value.
 		let bytes = ::rlp::encode(&val);
-		let new_val: T = ::rlp::decode(&bytes);
+		let new_val: T = ::rlp::decode(&bytes).unwrap();
 		assert_eq!(val, new_val);
 
 		// check as list containing single value.

ethcore/private-tx/src/lib.rs

@@ -66,6 +66,7 @@ pub use error::{Error, ErrorKind};
 
 use std::sync::{Arc, Weak};
 use std::collections::{HashMap, HashSet};
+use std::time::Duration;
 use ethereum_types::{H128, H256, U256, Address};
 use hash::keccak;
 use rlp::*;
@@ -78,7 +79,7 @@ use ethcore::executed::{Executed};
 use transaction::{SignedTransaction, Transaction, Action, UnverifiedTransaction};
 use ethcore::{contract_address as ethcore_contract_address};
 use ethcore::client::{
-	Client, ChainNotify, ChainMessageType, ClientIoMessage, BlockId, CallContract
+	Client, ChainNotify, ChainRoute, ChainMessageType, ClientIoMessage, BlockId, CallContract
 };
 use ethcore::account_provider::AccountProvider;
 use ethcore::miner::{self, Miner, MinerService};
@@ -148,8 +149,8 @@ impl Provider where {
 		encryptor: Box<Encryptor>,
 		config: ProviderConfig,
 		channel: IoChannel<ClientIoMessage>,
-	) -> Result<Self, Error> {
-		Ok(Provider {
+	) -> Self {
+		Provider {
 			encryptor,
 			validator_accounts: config.validator_accounts.into_iter().collect(),
 			signer_account: config.signer_account,
@@ -161,7 +162,7 @@ impl Provider where {
 			miner,
 			accounts,
 			channel,
-		})
+		}
 	}
 
 	// TODO [ToDr] Don't use `ChainNotify` here!
@@ -242,50 +243,6 @@ impl Provider where {
 		Ok(original_transaction)
 	}
 
-	/// Process received private transaction
-	pub fn import_private_transaction(&self, rlp: &[u8]) -> Result<(), Error> {
-		trace!("Private transaction received");
-		let private_tx: PrivateTransaction = Rlp::new(rlp).as_val()?;
-		let contract = private_tx.contract;
-		let contract_validators = self.get_validators(BlockId::Latest, &contract)?;
-
-		let validation_account = contract_validators
-			.iter()
-			.find(|address| self.validator_accounts.contains(address));
-
-		match validation_account {
-			None => {
-				// TODO [ToDr] This still seems a bit invalid, imho we should still import the transaction to the pool.
-				// Importing to pool verifies correctness and nonce; here we are just blindly forwarding.
-				//
-				// Not for verification, broadcast further to peers
-				self.broadcast_private_transaction(rlp.into());
-				return Ok(());
-			},
-			Some(&validation_account) => {
-				let hash = private_tx.hash();
-				trace!("Private transaction taken for verification");
-				let original_tx = self.extract_original_transaction(private_tx, &contract)?;
-				trace!("Validating transaction: {:?}", original_tx);
-				// Verify with the first account available
-				trace!("The following account will be used for verification: {:?}", validation_account);
-				let nonce_cache = Default::default();
-				self.transactions_for_verification.lock().add_transaction(
-					original_tx,
-					contract,
-					validation_account,
-					hash,
-					self.pool_client(&nonce_cache),
-				)?;
-				// NOTE This will just fire `on_private_transaction_queued` but from a client thread.
-				// It seems that a lot of heavy work (verification) is done in this thread anyway
-				// it might actually make sense to decouple it from clientService and just use dedicated thread
-				// for both verification and execution.
-				self.channel.send(ClientIoMessage::NewPrivateTransaction).map_err(|_| ErrorKind::ClientIsMalformed.into())
-			}
-		}
-	}
-
 	fn pool_client<'a>(&'a self, nonce_cache: &'a RwLock<HashMap<Address, U256>>) -> miner::pool_client::PoolClient<'a, Client> {
 		let engine = self.client.engine();
 		let refuse_service_transactions = true;
@@ -298,11 +255,6 @@ impl Provider where {
 		)
 	}
 
-	/// Private transaction for validation added into queue
-	pub fn on_private_transaction_queued(&self) -> Result<(), Error> {
-		self.process_queue()
-	}
-
 	/// Retrieve and verify the first available private transaction for every sender
 	///
 	/// TODO [ToDr] It seems that:
@@ -346,73 +298,6 @@ impl Provider where {
 		Ok(())
 	}
 
-	/// Add signed private transaction into the store
-	/// Creates corresponding public transaction if last required singature collected and sends it to the chain
-	pub fn import_signed_private_transaction(&self, rlp: &[u8]) -> Result<(), Error> {
-		let tx: SignedPrivateTransaction = Rlp::new(rlp).as_val()?;
-		trace!("Signature for private transaction received: {:?}", tx);
-		let private_hash = tx.private_transaction_hash();
-		let desc = match self.transactions_for_signing.lock().get(&private_hash) {
-			None => {
-				// TODO [ToDr] Verification (we can't just blindly forward every transaction)
-
-				// Not our transaction, broadcast further to peers
-				self.broadcast_signed_private_transaction(rlp.into());
-				return Ok(());
-			},
-			Some(desc) => desc,
-		};
-
-		let last = self.last_required_signature(&desc, tx.signature())?;
-
-		if last {
-			let mut signatures = desc.received_signatures.clone();
-			signatures.push(tx.signature());
-			let rsv: Vec<Signature> = signatures.into_iter().map(|sign| sign.into_electrum().into()).collect();
-			//Create public transaction
-			let public_tx = self.public_transaction(
-				desc.state.clone(),
-				&desc.original_transaction,
-				&rsv,
-				desc.original_transaction.nonce,
-				desc.original_transaction.gas_price
-			)?;
-			trace!("Last required signature received, public transaction created: {:?}", public_tx);
-			//Sign and add it to the queue
-			let chain_id = desc.original_transaction.chain_id();
-			let hash = public_tx.hash(chain_id);
-			let signer_account = self.signer_account.ok_or_else(|| ErrorKind::SignerAccountNotSet)?;
-			let password = find_account_password(&self.passwords, &*self.accounts, &signer_account);
-			let signature = self.accounts.sign(signer_account, password, hash)?;
-			let signed = SignedTransaction::new(public_tx.with_signature(signature, chain_id))?;
-			match self.miner.import_own_transaction(&*self.client, signed.into()) {
-				Ok(_) => trace!("Public transaction added to queue"),
-				Err(err) => {
-					trace!("Failed to add transaction to queue, error: {:?}", err);
-					bail!(err);
-				}
-			}
-			//Remove from store for signing
-			match self.transactions_for_signing.lock().remove(&private_hash) {
-				Ok(_) => {}
-				Err(err) => {
-					trace!("Failed to remove transaction from signing store, error: {:?}", err);
-					bail!(err);
-				}
-			}
-		} else {
-			//Add signature to the store
-			match self.transactions_for_signing.lock().add_signature(&private_hash, tx.signature()) {
-				Ok(_) => trace!("Signature stored for private transaction"),
-				Err(err) => {
-					trace!("Failed to add signature to signing store, error: {:?}", err);
-					bail!(err);
-				}
-			}
-		}
-		Ok(())
-	}
-
 	fn last_required_signature(&self, desc: &PrivateTransactionSigningDesc, sign: Signature) -> Result<bool, Error>  {
 		if desc.received_signatures.contains(&sign) {
 			return Ok(false);
@@ -656,6 +541,134 @@ impl Provider where {
 	}
 }
 
+pub trait Importer {
+	/// Process received private transaction
+	fn import_private_transaction(&self, _rlp: &[u8]) -> Result<(), Error>;
+
+	/// Add signed private transaction into the store
+	///
+	/// Creates corresponding public transaction if last required signature collected and sends it to the chain
+	fn import_signed_private_transaction(&self, _rlp: &[u8]) -> Result<(), Error>;
+}
+
+// TODO [ToDr] Offload more heavy stuff to the IoService thread.
+// It seems that a lot of heavy work (verification) is done in this thread anyway
+// it might actually make sense to decouple it from clientService and just use dedicated thread
+// for both verification and execution.
+
+impl Importer for Arc<Provider> {
+	fn import_private_transaction(&self, rlp: &[u8]) -> Result<(), Error> {
+		trace!("Private transaction received");
+		let private_tx: PrivateTransaction = Rlp::new(rlp).as_val()?;
+		let contract = private_tx.contract;
+		let contract_validators = self.get_validators(BlockId::Latest, &contract)?;
+
+		let validation_account = contract_validators
+			.iter()
+			.find(|address| self.validator_accounts.contains(address));
+
+		match validation_account {
+			None => {
+				// TODO [ToDr] This still seems a bit invalid, imho we should still import the transaction to the pool.
+				// Importing to pool verifies correctness and nonce; here we are just blindly forwarding.
+				//
+				// Not for verification, broadcast further to peers
+				self.broadcast_private_transaction(rlp.into());
+				return Ok(());
+			},
+			Some(&validation_account) => {
+				let hash = private_tx.hash();
+				trace!("Private transaction taken for verification");
+				let original_tx = self.extract_original_transaction(private_tx, &contract)?;
+				trace!("Validating transaction: {:?}", original_tx);
+				// Verify with the first account available
+				trace!("The following account will be used for verification: {:?}", validation_account);
+				let nonce_cache = Default::default();
+				self.transactions_for_verification.lock().add_transaction(
+					original_tx,
+					contract,
+					validation_account,
+					hash,
+					self.pool_client(&nonce_cache),
+				)?;
+				let provider = Arc::downgrade(self);
+				self.channel.send(ClientIoMessage::execute(move |_| {
+					if let Some(provider) = provider.upgrade() {
+						if let Err(e) = provider.process_queue() {
+							debug!("Unable to process the queue: {}", e);
+						}
+					}
+				})).map_err(|_| ErrorKind::ClientIsMalformed.into())
+			}
+		}
+	}
+
+	fn import_signed_private_transaction(&self, rlp: &[u8]) -> Result<(), Error> {
+		let tx: SignedPrivateTransaction = Rlp::new(rlp).as_val()?;
+		trace!("Signature for private transaction received: {:?}", tx);
+		let private_hash = tx.private_transaction_hash();
+		let desc = match self.transactions_for_signing.lock().get(&private_hash) {
+			None => {
+				// TODO [ToDr] Verification (we can't just blindly forward every transaction)
+
+				// Not our transaction, broadcast further to peers
+				self.broadcast_signed_private_transaction(rlp.into());
+				return Ok(());
+			},
+			Some(desc) => desc,
+		};
+
+		let last = self.last_required_signature(&desc, tx.signature())?;
+
+		if last {
+			let mut signatures = desc.received_signatures.clone();
+			signatures.push(tx.signature());
+			let rsv: Vec<Signature> = signatures.into_iter().map(|sign| sign.into_electrum().into()).collect();
+			//Create public transaction
+			let public_tx = self.public_transaction(
+				desc.state.clone(),
+				&desc.original_transaction,
+				&rsv,
+				desc.original_transaction.nonce,
+				desc.original_transaction.gas_price
+			)?;
+			trace!("Last required signature received, public transaction created: {:?}", public_tx);
+			//Sign and add it to the queue
+			let chain_id = desc.original_transaction.chain_id();
+			let hash = public_tx.hash(chain_id);
+			let signer_account = self.signer_account.ok_or_else(|| ErrorKind::SignerAccountNotSet)?;
+			let password = find_account_password(&self.passwords, &*self.accounts, &signer_account);
+			let signature = self.accounts.sign(signer_account, password, hash)?;
+			let signed = SignedTransaction::new(public_tx.with_signature(signature, chain_id))?;
+			match self.miner.import_own_transaction(&*self.client, signed.into()) {
+				Ok(_) => trace!("Public transaction added to queue"),
+				Err(err) => {
+					trace!("Failed to add transaction to queue, error: {:?}", err);
+					bail!(err);
+				}
+			}
+			//Remove from store for signing
+			match self.transactions_for_signing.lock().remove(&private_hash) {
+				Ok(_) => {}
+				Err(err) => {
+					trace!("Failed to remove transaction from signing store, error: {:?}", err);
+					bail!(err);
+				}
+			}
+		} else {
+			//Add signature to the store
+			match self.transactions_for_signing.lock().add_signature(&private_hash, tx.signature()) {
+				Ok(_) => trace!("Signature stored for private transaction"),
+				Err(err) => {
+					trace!("Failed to add signature to signing store, error: {:?}", err);
+					bail!(err);
+				}
+			}
+		}
+		Ok(())
+	}
+}
+
 /// Try to unlock account using stored password, return found password if any
 fn find_account_password(passwords: &Vec<String>, account_provider: &AccountProvider, account: &Address) -> Option<String> {
 	for password in passwords {
@@ -667,7 +680,7 @@ fn find_account_password(passwords: &Vec<String>, account_provider: &AccountProv
 }
 
 impl ChainNotify for Provider {
-	fn new_blocks(&self, imported: Vec<H256>, _invalid: Vec<H256>, _enacted: Vec<H256>, _retracted: Vec<H256>, _sealed: Vec<H256>, _proposed: Vec<Bytes>, _duration: u64) {
+	fn new_blocks(&self, imported: Vec<H256>, _invalid: Vec<H256>, _route: ChainRoute, _sealed: Vec<H256>, _proposed: Vec<Bytes>, _duration: Duration) {
 		if !imported.is_empty() {
 			trace!("New blocks imported, try to prune the queue");
 			if let Err(err) = self.process_queue() {

ethcore/private-tx/tests/private_contract.rs

@@ -74,7 +74,7 @@ fn private_contract() {
 			Box::new(NoopEncryptor::default()),
 			config,
 			io,
-	).unwrap());
+	));
 
 	let (address, _) = contract_address(CreateContractAddress::FromSenderAndNonce, &key1.address(), &0.into(), &[]);
 

ethcore/res/ethereum/classic.json

@@ -57,7 +57,8 @@
 		"enode://814920f1ec9510aa9ea1c8f79d8b6e6a462045f09caa2ae4055b0f34f7416fca6facd3dd45f1cf1673c0209e0503f02776b8ff94020e98b6679a0dc561b4eba0@104.154.136.117:30303",
 		"enode://72e445f4e89c0f476d404bc40478b0df83a5b500d2d2e850e08eb1af0cd464ab86db6160d0fde64bd77d5f0d33507ae19035671b3c74fec126d6e28787669740@104.198.71.200:30303",
 		"enode://39abab9d2a41f53298c0c9dc6bbca57b0840c3ba9dccf42aa27316addc1b7e56ade32a0a9f7f52d6c5db4fe74d8824bcedfeaecf1a4e533cacb71cf8100a9442@144.76.238.49:30303",
-		"enode://f50e675a34f471af2438b921914b5f06499c7438f3146f6b8936f1faeb50b8a91d0d0c24fb05a66f05865cd58c24da3e664d0def806172ddd0d4c5bdbf37747e@144.76.238.49:30306"
+		"enode://f50e675a34f471af2438b921914b5f06499c7438f3146f6b8936f1faeb50b8a91d0d0c24fb05a66f05865cd58c24da3e664d0def806172ddd0d4c5bdbf37747e@144.76.238.49:30306",
+		"enode://83b33409349ffa25e150555f7b4f8deebc68f3d34d782129dc3c8ba07b880c209310a4191e1725f2f6bef59bce9452d821111eaa786deab08a7e6551fca41f4f@159.89.223.6:30303"
 	],
 	"accounts": {
 		"0000000000000000000000000000000000000001": { "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } },

ethcore/res/ethereum/ellaism.json

@@ -13,9 +13,9 @@
         "eip150Transition": "0x0",
         "eip160Transition": "0x0",
         "ecip1017EraRounds": 10000000,
-
         "eip161abcTransition": "0x7fffffffffffffff",
-        "eip161dTransition": "0x7fffffffffffffff"
+        "eip161dTransition": "0x7fffffffffffffff",
+        "eip100bTransition": 2000000
       }
     }
   },
@@ -29,7 +29,12 @@
     "chainID": "0x40",
     "eip155Transition": "0x0",
     "eip98Transition": "0x7fffffffffffff",
-    "eip86Transition": "0x7fffffffffffff"
+    "eip86Transition": "0x7fffffffffffff",
+    "wasmActivationTransition": 2000000,
+    "eip140Transition": 2000000,
+    "eip211Transition": 2000000,
+    "eip214Transition": 2000000,
+    "eip658Transition": 2000000
   },
   "genesis": {
     "seal": {
@@ -60,6 +65,10 @@
     "0000000000000000000000000000000000000001": { "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } },
     "0000000000000000000000000000000000000002": { "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
     "0000000000000000000000000000000000000003": { "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
-    "0000000000000000000000000000000000000004": { "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }
+    "0000000000000000000000000000000000000004": { "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
+    "0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": 2000000, "pricing": { "modexp": { "divisor": 20 } } } },
+    "0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": 2000000, "pricing": { "linear": { "base": 500, "word": 0 } } } },
+    "0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": 2000000, "pricing": { "linear": { "base": 40000, "word": 0 } } } },
+    "0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": 2000000, "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }
   }
 }

ethcore/res/ethereum/foundation.json

@@ -174,8 +174,8 @@
 		"stateRoot": "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544"
 	},
 	"hardcodedSync": {
-		"header": "f90219a061d694007fbaca6e23e73e29c8c6a60099abc740ab7e27ae3cd1b9c6a47efef7a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347945a0b54d5dc17e0aadc383d2db43b0a0d3e029c4ca0a2f1bdabc1a72737de1c22a76cacc8fc162366904f759a99db7d6d19efee3090a0ac5f5b236e8977928a2ce43c7569ea5a74919643cb0b06d7540407b1ea1298f0a04356ddc5d77c83923a6541260308be167386e0969a608a017770c9e38091cfcab90100a00010002001009080011010141088000004000080081100000a002023000002204204801204084000c000010008000000000880080020c0000440200460000290005010c01c80800080004800100406003380000400402040000028084002a80087000008090a00200100544020019580022000000306100a0080100084020006809000e80000010000254810002000000a240050014200002002c10809202030006422022000203012000241089300080400000009001021020200012410348500028290230408100302000000058c0000020c08c20480081040020260004008481000080000800010010060020000e00020002140100a8988000004400201870b9af4a66df8038350a8018379d54483799eba845ab0426d984554482e45544846414e532e4f52472d3231323134313232a05eeccc80302d8fecca48a47be03654b5a41b5e5f296f271f910ebae631124f518890074810024c6c2b",
-		"totalDifficulty": "3144406426008470895785",
+		"header": "f90207a0cdaf426b80edd4363b336b351cbfec8c38b44847cdbd814aa92e92bc9ec05333a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347949435d50503aee35c8757ae4933f7a0ab56597805a03f28b2b384dbfd29bc0a10343e8a419e61b92782f880046170bf1d11455e94bba0f3712ef3ff24efe1afc7da11ffb2ca495a94fe7958f42e9f1599d66ed72af13ba06d01d03a15f807da601bd2dfde7490bbe91d9bb11c11eda435db9daad6e7b1efb9010000008c0000c000000000440100108040000082000800000000000000040801001004001000000010000000001000043300001000008000800000002000200040000000580c0004108000040c0008000006000000000280800080000800000000402000000a000000a810226200002881004000208006020000000510030000100040010100000086c20000000009000100000190008c80060000008000202080420008056040000000001000400001100010140822800220000c804004002000108000160001400200088082008000000412100010080205011000000800a0000810021005000000000002840000000400000000880000006000000000200002870bfca554dbc6398358b001837a137083473c8e845b27f4fd86436f72746578a0dbf31fd28bd8f69f1103196e1782a9dfb636bcfa726362ab0767235cb8d56e7188264402501145497f",
+		"totalDifficulty": "4838383800145139949936",
 		"CHTs": [
 			"0x0eb474b7721727204978e92e27d31cddff56471911e424a4c8271c35f9c982cc",
 			"0xe10e94515fb5ffb7ffa9bf50db4a959b3f50c2ff75e0b8bd5f5e038749e52a11",
@@ -2757,20 +2757,274 @@
 			"0x568f44291c13efc908db42d2473bc91ebb16e062e9b4368bcb770a3033d67741",
 			"0xe5ecad510448855ff0aafb92a8c7aa54aca0fb390bec3c14ad5d2ba200380aec",
 			"0xa40aa7655c1458b76c04ea5934ae171fb01c72c8c375bb61a0c27b0ebd70f21f",
-			"0x770ad5107ac47fd983979b016553ca8f340a13e2647b9140e65c980dcf349cff"
+			"0x770ad5107ac47fd983979b016553ca8f340a13e2647b9140e65c980dcf349cff",
+			"0xfd074cacab08d2d5f113669672632ab0e94e03bfbbe17fc3de27a30e9f0e8327",
+			"0x2063bf5ce8eed483242157113534b314296deb123cb33083cb0cce04db610c9a",
+			"0x9dfb8274e842fd3cb7d73506e64adacc39eb12ebd8a972b7101385ac4eb5b12c",
+			"0x8294c3d4892cb0bcba9acdd31d53af5bfc97daf124f08d903297da8bbb28cdfe",
+			"0x9dc2d04fe197009f24ebd4a0c95a0b91d4ff0093387b61569af993fa91edb3fa",
+			"0xe88e256abdc6447b1f141c2fcf2c46721d9717a9fcb302513c93ed6e31b7c11c",
+			"0xd2ad4ca6091f6ae6c323b6565522c208a839e36958a1675e5e9b48b13068e7c8",
+			"0xb168169f2643b8cdbec58f3f94d7889af37117a35389726563ac770f5205faab",
+			"0x09f32b423ca728b8fd04f9872325c0e2a4c9b1eb36fab49b31ae51254c69ebab",
+			"0x0b47f8f28a3242c9ca705fd11949e648f190053de023366f98e6c736eb2f0de1",
+			"0x870a8067623b35b5f0146ec95047869357617fe3773a8285a622c18d44963f9f",
+			"0xd14e05d8f3238294ffd8885c1f1710f64f2d8eb3d3b4e8a921f7f5c94a0a6288",
+			"0xf9cdef6ffacbea7692e9fef3de1b778bd11399492486dca0209b27bd474eb04a",
+			"0x5dc41201eaa00957d070c68bb289fc00eb31b7e5fff52f0603d52d27a40abf81",
+			"0xae3a53054ff4e3e5562795b45777e1e3e82abeed84e2fab748af50aace8bc8e2",
+			"0x93fd6f9af3b1efd27487477edccd690b19902300a4eebd34cb0d9b2e60b3cccf",
+			"0xbe475a9dc045e70ac8218e5a65450edf026d70f3f814162251f83a88098245c2",
+			"0x20566435d247cc1e7d0d81e143c01b6649fa1fd3cd5a9f97738e6045b29460c5",
+			"0xb37273228fe8b0c1679bb2dab7176ee1282400daddaaeb0db415c716f5dd1f71",
+			"0x494bb2cea59fc43629b0f8be3fba9ee5b52948c72fd1f1c94d31600756bfb675",
+			"0xc690eb3d8e65d5efc211def8fe8f198e42c0321000b8998a6ab218d897d593a5",
+			"0x26821d73690b9d1bdd9594c92b086291474cd365bd41f7de5edc49dfc29c26df",
+			"0x026a57937338c9e168d254f99dc9dbe3fb41508049ac1a5b6d7df868e2935e6c",
+			"0x50dc451812556bd69244f3728c6a17e958b2fa125248103942ac94142610bcc7",
+			"0x0ffe7ea32c3ef43049d8041a16bd3381ba89b1439dc5329e839546aa01a64726",
+			"0xab1e06f13dbaed341fc9e33ef387ebbe187787ccd8961a47560f7e02617a06db",
+			"0x8b72166b0d16ebfd9c3156f5972e3778e16b3d7edf3decd4b6efbc779990ca3c",
+			"0x149de0dcdbe03bf6d0ae9ccc989a074a4cf2e78311ce8cb6dee3856c2a374caa",
+			"0x42147f5960814dc31cc38b692089e6d1d53b85cc85a69e4cc01954dc61c268a6",
+			"0x1460ceccff1ac00f3e126383c3102c424b75c2abeb973d0baa8bec8e7bdda692",
+			"0xdbbbf28a5d25c832c5f625868dea79a481dce880d5145f4119d3e9eb32356a6b",
+			"0xd8e5b32a88bdb502fbd62888b9906a0c7955ccaaad1ead31636572a07fe7419b",
+			"0xdbc42abbec7d223666da11767d9862cd1458f91b5bf2f6499da4ae5bff281888",
+			"0xc3138499753f545de139238bdc8cd4cbad254c6b87bf0eb37dc1fe7a85168e79",
+			"0x42be700f627bb081543ab289fcd01235bedea5e33a9779d6d14744685f5a652a",
+			"0x8660e4130c98f6a8024c1b587dd24ebe402126d4d9a2adf610c62bf033b749c5",
+			"0xdab926bab8c4ff9e492f992b5696e8f7e3ba3ce77e49fe0e2ca36d3efc8791b6",
+			"0x0e3911bcf379fd57014bee6a1f796b884c62f2392cf6e8ac5a145126e4622b88",
+			"0xbd52346714664eeb7b4034bc6a54804824bdd40d9e5bdbae9375a302926babe6",
+			"0x641e58eaeb404f7a40b9c4be686c26721563b310f7624a654185a60374c1512e",
+			"0xfa07b0f7b970413ae5085f042224861bb1bc5f2f6ad72a44c3681d47a817c0a9",
+			"0x70cf3fffa7e27f03d5cbb1664bbd0e1315dfc43df76f1b0e3dce494255e54752",
+			"0x1d5b0118939994aa8be89dc365f909309df84e7c92530aef25d62118b09e0572",
+			"0x8b34c5a5cd5a42a58bbc60166f3f2a4be7d79057691c708e7a9639c21b6f57c7",
+			"0x7b149005e7f8378189c70c33c073d897cb07f62ba5ae319088b3be204beea8e4",
+			"0xf2a41e8feb5d294faca47dd1f7a90f66b7b616addcd179ae60c5f173cd44a85d",
+			"0xa2875c8e5a226f2cf69f07e19b13a628ef6a8662f3a5de4c9d666ca5ebddb001",
+			"0x4231ca8bdccd59526900a487608912a4fc0918a6c693250bd67787bc61a9d09b",
+			"0x2b06301a6372e0fca0da8a15d4e31a08522ecb91efbda58c3e495e2ab7f4fceb",
+			"0xd0e19f62b07275097b56e7f409cd52826a1dbc0be26eeaae20562420559153dc",
+			"0xa4ecf1ef04bcc95ba0ed9214e9118129ae5497c37db94dda284dd536b4dd75c9",
+			"0x3dc87ba482aaac6b59678c899bfb1da9b768e1385e0f8e8574852efeee2db8c0",
+			"0x07f8df3f6ec63d926c3a76b17ad1750a9ac6c62a8cdffa549440e651f6087178",
+			"0x96952f8ba7c83d640119868a030cb7397624ef322b2d153aa3843259da5f468c",
+			"0x08eca2fb4295f069f3fa62849b354a9e10b8a8e9b771cb8d2790b28b7b09d70c",
+			"0xfd5832f531c8819c0ce9d031434c33eb442578e81cf711e8ba23e9b21d096e13",
+			"0xfee9a8bae124565c963b65fa47dcf7f0c9b5ef7e778c2d99539b8a04efd04522",
+			"0xead692739a0723a73dd38b3aa02f93d72b3b6922c2520ea0b74981eaafe7f6a1",
+			"0x890dd8befeee6c7c7b4942b02a3bf1a9d1b9e1140ecd1c07de570f38b6c323b6",
+			"0x690343489ed214b9195db6181c42237b7a62fc935a18ca1a5dc883a2045ceae7",
+			"0x4bdfdc1bef8c19b89344230c0f48841abfeea788a4973c34afff4d2039ce4417",
+			"0x6ef6a94338d794400f95f8c0b6d5a40df8b56c31e9bd0037bdd7e41234bb32a1",
+			"0x00b7568bfdf2d4780b8650a49f707752a12fe9a88302622e68f264622877acdc",
+			"0x83e96e0b864df30b53ae08075acc83f785e9b8b2bc80e8976b1a954f995df012",
+			"0x664305b610d0b3167dd25210acba7eea291a37a224546a6f6d59aa7b71d16e5f",
+			"0x3d3a6401ab83eabc2af15a23b9c0e6d123f661b3599c6131d5923a106f562ac8",
+			"0x7a11833fbc8ce3d9883088649b930238625e6f6136dfa06a00daf8e646cd020c",
+			"0x112c33932097c4ec4e18f6308b2a6502cbf746605f1e404555dd07703d60ba2d",
+			"0x9e77cd028749d69caf352a95f75dee842c0d9323e3618afa2927b2aaaf7fc4a8",
+			"0x7fe0a62bfd04e263b4c67c5808f6231ad6faca7ff0f5bc0c02177034608d6991",
+			"0xea820d62b423b68da2f8ce345dbb56e2205fdfae1fce261d64acc959fb30cccd",
+			"0xd8f6367448dec55e8e2e26b5d304d92969abde131d402de313b1705c0bfd528f",
+			"0x4b97f3b4e0bff7cfe322e4f50176f6bfb379a706d42308ae75b2a46735762a95",
+			"0x7425ff9cea5e4869332ce12ef727fbcb004add827eb567424fd4c17717def390",
+			"0xc7dcd4fd005ae8d4d9e125727b54a64e900b3fee9cdc82fb5057c012178c8483",
+			"0x8ba88bfba201db1db716f762b5517369d3ae232ca045d2932fc27f6c47b15806",
+			"0xad2f63fa4fddd0cfc7f6cb01116d285e8aad4b0c51fa9445d39a4cb4949ed020",
+			"0x366874aec4ea26ece424f5402604ec7df9f40fc4cb9087cd3f45e764b1e36ebf",
+			"0x7f27eb75010b0d5da72794c129042511e24794b6c8491c1ff2e257dadcc7052d",
+			"0x27eadc596f6eaeb9247704c3339f7fe4e649f683fd815f9d71270caf5d9e38cd",
+			"0xba3f72ce8f45b1575554bd0c64feda3959c2a68f0300f021b67880b56f7152e2",
+			"0x50833c82dc63533f7ec918cd6d58ffbef4e2d597f354589b8eb66c9de0fc9697",
+			"0x30fc354a8893f683ef03bb024254574b550710f3c05496976cd39166a29e1c98",
+			"0x1b6b8d13fca6583d358b173c1a255c3db9efed8ad72084eadf10e29868a26fdc",
+			"0xb1b2a80122d1210160d8649d31422a1edc26b23e8499aa3a88b047dc52b42222",
+			"0x4e6c85a13cc0c739fbdf844a4f52c78979a84a86db728e4a5d2043ee3b2fcb3e",
+			"0xe98c28b49aa690aecfa43527acd3e5a010c8f90faf79ef5b4b479a715fe66a99",
+			"0x5b56e2b50ec96f7bda3f529ed71321650d688e2ee5c16723f6199256532ee660",
+			"0x1c9b8f5106a23096f2f57dfdb2a7a82876f4374be97b6a818fdc3c742b216b09",
+			"0xcb6973f775356ec7596ed6f9a036258e98677c7c5b2358da75af28f553327506",
+			"0x26b42a54252893f59e8eca923a0d2f2c0fe507a680e8907e2904702791c42aea",
+			"0x25fec26921bb5b6cd9657588b09813270cad72947abc6cb2726764db44181ff2",
+			"0xdfbd186df632b63f4cf74db660c59e658e0b4b725fe244a563c6698dd94adaf4",
+			"0xd1e00635e2399f6fa625f5c204e4c1643a7b619271394141433d26d902479bbe",
+			"0x3d3323fea45851eb0bc0b6c80a79cc0d0582abd8032aba957e719c31bb7901e6",
+			"0xe7d7b4c68d4f55ea8b71b43ad20bdd86402d36b6360ed0ca18082942736a7e41",
+			"0x1436749eca0a72b74bf8d596e3499414d412fbea0b936ee3564c4f83de907092",
+			"0xa828c16af52bd4adcb64928fada82a02d0a49fd3f53c5083ca9acfd998f8af1d",
+			"0x0fe559ad45cde755dd41876c5a65a690d32fc3d294fafeaa43e3fe133ae48da8",
+			"0x8f91d2082e5a1e6c1c905936515ab5f03889ac67261ef1d892fd909a9379a8ba",
+			"0x3881a9fe4ba6a24dcfa86351be2ca741dc9b2078409b2e7e0140569f19733505",
+			"0x4a3295525bfdda29bb9a552d8dc5c992649699f5694f96ff5bb0647910386db2",
+			"0x337389b3e800bae87fdbe5271f2167f169ffeb4710ecdcea30f08b2cefba57b1",
+			"0x2978e1e3c2b5dfe0b41ceb5c9c4029f42d346a2123a199c52ba6efdbf1d5fb68",
+			"0x8abbdb4f1f88fe8900afdfe15f3d03244435d4fb871606a689c11f6420145b45",
+			"0x986dd2ca80e46f37a604b7186ce5683d3e5e114384ed6ccc39102228fbdd0eaa",
+			"0x27fed9eeaab228907e106872d14473d621559176063f3c19c98393215ec87e02",
+			"0xb743e115e42ff9ea8240082a5fcc9f5878221ab742d0db8f1b0edc6317484f30",
+			"0xf6d2172dff821efc35df518767fbbf59eac9fcbbf6d89246bf845b588f5277ab",
+			"0x1c608dc9a9114a38c14d81a75c038efb591259dbecdf00a41f0bf402f4dbbd92",
+			"0x0ec49c3e2e0617fa336c1f9e36bcdf6ed25e548281abf1ae0286c5d92571752a",
+			"0xcdd7dd5d936b3b685352a9797b52419c6f9aeadf33e5dc6dd8b0ff35a85d2a35",
+			"0x9bc44252049c89b94967579c0ab737e14dfbdc95eb23d14579a73f3e68a81def",
+			"0xe67fffbb7a6dec1a39c46e838d72f6bf7c48023a2759f2c3e4340bcbdba057fe",
+			"0x55008d64590ccbf16e1288121b2a885d4852dda11cdf4bc21c578f85fc7eca70",
+			"0x0ab47947ffc76c87f5a4d8c5eed162bf0b56b0cac11e425b44f270868f22fd8b",
+			"0x7f5810f6e39bca9e268e1d0f2fe8b3b172f60a84e51e61f2b9056854c2dfbfd1",
+			"0xea68e4e38860b5fbb0f509266afd110d50b6d34c7bc083f772878716469f7202",
+			"0x203cddd6d6a1ed1174ad13faf518d08a11bca1ed4809bfd9d2e8947ff21664b7",
+			"0x00e1378e25cf45e210f0757da309b30328b3ba3a1d270a065bf797acfd696eaa",
+			"0x4327597fcde5e099327889cb075d1db345f5fc996888b058d3d17a0e49708e51",
+			"0x6ca6195a4887f9dd5112621a13fa67463bf945d7aea39578feb1a1107c88eb79",
+			"0x5ea64f4ead2b127c29cb51c79b3c28b81e345e2eecd237ef57fde7adb8702972",
+			"0xb2c8f6e30f4b70a2983b508502cb50e6150fdac3b6d60c8458f9bc3e9b0c1fdb",
+			"0xdcb684901513685c0bb7582aed9ee01b98297e05e9b054f3ad6917515e7a8620",
+			"0xab7ac631c98ebba065ca7d6d81b58e8cee0d1304b6828ed4699f07ff2d7b0f18",
+			"0x21f12ed6ecd4fcb2455ce8472f01ef78081fe3f3d4eee5a7edc7b546bec5e01d",
+			"0x0735e851626195dd9baacc806903c5a623eb3d3902ac2514db9b088292f6ce31",
+			"0xf8008dd2e9b354b284425bca8b378fd2081449c30cd6bdec3a8859b0021953c2",
+			"0x809f2803965be44e3dc219d279acd5b3bee6c25780cd498d15dae3325d8d568e",
+			"0x5fe269bcbeb9d2d2b253cdf4f0f2bf669c93be6cf428996b2137b68911ea0724",
+			"0x41e04646fcf4a232b331a15619978ecfb13e9095ef266edd812aa1793e1a8a54",
+			"0x0553fdd5e662e92852697a7f41679b1a82940b5d2acdea812b0925b7ea3892b9",
+			"0x6e77324bca3ecd57316541eeb110f007a06359dab94131d98c245c0a137015ce",
+			"0x217b4e8911ca232af5cf1673679137cd720e530dd704d11987eb8fbd612a8b16",
+			"0x42bb0c8c8d5a2c49dce7bc4ed6f97869fe1335d3ac3f8df89d4701ebe8a609e9",
+			"0x2cefcc4dfddaa49857629290b1b97af2ed03ef3b8e3fc67271315c6961038972",
+			"0x6db8fc507494b4b5b6d7c95e056fe15571cee06adf6e7d21dbe4c5d1787b9bd3",
+			"0xaf5c85ac940764c2e50ec0cf57c37febdbf1e4539dbccea24e7e0dc499e4b38a",
+			"0x063f930786584fec4dc4aca21922770de0547b455e437a7b0b245edb9d161005",
+			"0x94dcc780bc2c40ede15992b2ef6667965dd346bcf00cba42e718d8f83c68f325",
+			"0xbf5f4beb305c3cbe0ef9142f3f588804f0437c9696fd2190c35face26db36c6e",
+			"0xe8ab9b3503167c91116debe3ce48d0d76e23ed1c1cd12da09876e4ee518a39f2",
+			"0x1fae4431c7874fa5d2787a520b27b8a0826f095398d7ca1e94b234650c04ea98",
+			"0x7f5d261d5c29232b5ed9c55a26a59fedda566fd3f6bfbaff11d63c4ff161f5fd",
+			"0x763d7abaa36d8262b659783a6a310e965a10b362e9bb7d907bef107a70f14e50",
+			"0x0dc66388e318c83fa3cfdd1136b621323e9d80cc2b97a84029bffd53eba93cf1",
+			"0xb422530c15aa6e8b4fa35535040e4b7b64de68a41937a439592be5d9afd4f699",
+			"0x3fe4f704e67fd2d9ac629ce20c622f1634deae9fbe0a42b02da21c4f119250b8",
+			"0xdd6ce24b6084576c646a2d8e42aeee919ecd2b2842964812c31c6c18adbe8676",
+			"0xad0b9133894fbeef5ca4562aff4caae74218922542f6b20f73f99a4e5f30096c",
+			"0x751d56d8e50e1f6b1473ceeed9ddf34030f0b203c9e705a8703a4988d1b0a3aa",
+			"0x777800d29fb6fd9f7fbab607b4d9852f50f2aa3bd52b7383f7cdc287771181bb",
+			"0xa34077e2a2e59e10649fe850757e3db69cf1fe0be2cd34fd4ac1a2793668f0d5",
+			"0xea7d03f2cbf31f66327d8dcc7eab05c9f7fe98517f3d88cedc1c590742829106",
+			"0x9c834993c9a684c8c726866c0a0013c3a92c275002137fa0288368268f513c8b",
+			"0x30134d868fdd3c9d3e17c19c2813605dd3d5fdc9867baac66312b3089f1931d6",
+			"0x35c44b2ab3def27f323b69a679c1536d8c4ca8e8c5dd33fb4db1c28214db48ea",
+			"0x4da76c0233f32f66d1117d436aab4e73a7c7d3675aa2fa0abbef5498973fbbca",
+			"0x02aa3319e1c7710604fc26a1622651dc319b5119a68ee4e887aba1b66f85e4a3",
+			"0x642722b31dd1cf026be151a590ae81c8886812d858159c1ea9c40fb93b88e73d",
+			"0x84942ffdfe4af44680244f7810239ce734981f2c9b6a42a951261a3955bd31fe",
+			"0x4bccaf057f4fc0914ff58b56f4ffefd096ee71c508d4e94f70d30f59a3c8ef07",
+			"0xfbb306bf85220f2a93533747e5d081281d323419adc3458327be9f50ce700dfe",
+			"0xd3ebe4dca06c2005fd62be1448f92d0bc8cbddec22564e3adadd11e4001432ee",
+			"0x4537177c35266327008bf899944a740739fce625930df5b89fec5cd43401c665",
+			"0x3346cfb4b3b5f6a2c34b478f823d5a5cea08bcd026d1dc1f8cf885064d6a696a",
+			"0x6d7d217e6f196549a651bb51500e693299a5838e23651902e54a36aa26d3cecb",
+			"0x03fc17e1d0df60cb54874c848627ea0a86fdd8eba989faf64fe5ec3ece8293d1",
+			"0x3a671486102a9dc591f67b937106a1dcee58680062a328f2fadf8fecd8fffd7d",
+			"0xdfa484a44d55c86e2c3b05429144bbb3e6758e31e12283226eee25d58fce1e04",
+			"0xe01ef4307108875450a351b5df82c5fea4a3d34a662cc87f7071cdabc7e5e4b6",
+			"0xd04e6b7fca513c797b05b49ca345a2102243b6e893407f45247b0b406fa7cc6b",
+			"0xc54725a8216701640e068f8a8ef8e77db86c7e30cf4add44f1fd125a6ac4dc11",
+			"0x10851f13c23fed1edd5434deeb6a07d466d922f1516a282c23ce7834faeaaf62",
+			"0xb4c3613b7c51412a171f1a2b36ffc01801abe4594b05ecc5eba6026139892f0f",
+			"0xccfa7a95e0a7eabc97d234da53c80e94b811f270bebb6f6ac8759657e3750e3a",
+			"0x3adf3ad22a66a6205a76c91d488b8f315c12df4955c52605724082d1f3b93da1",
+			"0x9cd530ef2e1e14f8003b8d8651802bd1b0a4dceba53dcedb93a115ce4ee127a5",
+			"0x803bf4629682228ddb883c5a81bce39a234768de6c299f65a07a3ee5895087e9",
+			"0x8b6af66caee2b87a3fd2643c72e8d84ec9c0c75fa271456c48cbde2774b2b134",
+			"0xc361af073d0a9044e2e28cace7fa6dbbcf7c05e41cc372ae13c2c5e3b1d42ea8",
+			"0xaf73a3936f76a401a3a137bb6479978acbcaff7b8b0f6758a72a0c772fc2ca1c",
+			"0x53259e3f0ed789a4397fb6897f768d526de648c4cfe50e4b88a95ea93232e404",
+			"0x2145a286266f525fcaefe549221f470a5b8472db76d35e820c62e9dd09f41ef3",
+			"0x9834dc8c6a845e685219a66157f77809c2dc52dbebdbf36a7a768aa4d2a89edb",
+			"0x300e7319a340f894b96281edb2697fe70c650d5e1a112760f22f10deb32ce547",
+			"0x0465a059e6b40cc6ac10a2bc78b6d064d6765c0adb0c20a45290e65560bd1e1b",
+			"0xe935237f4f93f5de4d7b768627ce849aae62d8a5d807e0398f142677165fc3ba",
+			"0x943f9abefc54fe4f118401b9682c44c9d7d97d6cf03ddf38d32cb694f9af1c32",
+			"0xf658af875a69361211c9894ff4a77e595026d4e815b3c18e9cb83774790a5cca",
+			"0xb5ace322f7b347062e8b728b312ebcddf6bb95dc24842c6ab783430cbd5e52a8",
+			"0x8ac46515d61b24567ee93b19a3bb407a14016ee4b3b99b954c74923504e5baf9",
+			"0xb7c2a9de470c3a43b7618fa31d4cc0b9ac879a2168ef6ed1348fc66c486340cd",
+			"0x9eb593e57bc9784336500e4a593200274abd0e9da1c8a16b39acdee94a0be676",
+			"0x361e998b3e24f213348db959f4cb65a04ffd1de7a705047fc10733034fbe1c66",
+			"0x47b2b5d68a3080d6f11ada3bf0af1d90759f8f08421151f6a2ed5af64974f262",
+			"0x002fe24eae8abd356bf631051897ac8ae1a754f7c5bdbd5fc521bcd364fdfd96",
+			"0xff53d1ae5e5902c1a1b66bc36f88f3264bb01f14dd84def0ea0f94c7c7eadec3",
+			"0x93813bf1b6a660f87410f862123d1ee694eedf4e7464a1a9e4556e3e3e2f9a18",
+			"0x077cf660cd2cc0f4c4cda9cec68d8dda36d7c95d04cdace3b70c8d010441a68e",
+			"0x367029ca5dc7cf420efed09c07561e9981b0dbd8dd1b1126ebb829cec9b5bafc",
+			"0xa111113ef2ef722bd548b5adf4d75ddea2a7530d4d843b022c2a54ec25ba8c41",
+			"0xfb2337d6125ede563770a84191a6f158ff45a00ad5006153765bbb72f51f6e83",
+			"0x1ede4c16ea063d9ab7e7962957b4c067b0fe25c2403deffa5b38af79c563aae9",
+			"0x067bc5f7409eac44326e4adced1eb688e3d9834310de73f141e90c4c63342dc9",
+			"0x909d8a9ea7bd84097eb6129b8609773b90296d9d01729f592b4bc50981d9f1f1",
+			"0x11f09438cd48f7895d90a6bb2dc9fdc342b8f11074d988b272f2c95b76f99fbf",
+			"0x7ed2111c775c418ac81e55886fdf8fefab1f639e243f47f2c5526ac85b3309fc",
+			"0x376919526e0fff65b9655bc34020d9df4c91300c0747b225ba9ac5e9adbf97c8",
+			"0xf6ae43cfd33c14899af2575283288239210f3c00de11364175c3a1556637830d",
+			"0x544063d4d8cc6b471b354b0e57773d841052ecc0110870b37f6e8657d4256381",
+			"0xc1f6d5b592da819a2ebaea65a0a6448f63ce09fdfcf1c15f93fa20fb8b67e074",
+			"0xc337d4acdae3bbbe7275c21b07b04b2ccf72a87b5666ee002ed54b7f56c6a276",
+			"0x8956d8aed9c2ae1b76b1aeec8210cd24cd900ad5fb14cf0e86e86ba35f5b04f6",
+			"0x06219a3301edd50166ff67184586c2caf2b3553b73e863cddd7a3c35b7f88022",
+			"0x52b99f92dd3ffa353c0b54466fea2bd1876b3af2b64953c12cbbabc266197e04",
+			"0x5c69f7312b8786144aa2161b3c965cbef42cfd41c8b586d117336ae8717b65f5",
+			"0xe57747520a141a1be0140ce4b45e2973479ee8f4a50a85162dcdcf5090380528",
+			"0x1f73420c464ceb6bd25e2eadff6a248ba092dbbeb5b3046ee809567dff1f34cc",
+			"0x1df7ccc998af1e233e61adeef0f757caa4a80270eb6d5be5e920d6b085a4cfc7",
+			"0x514024457e175885e55034f5c28c666611ce6bc1b8be8c3f253028f24091672e",
+			"0x7574d79098c1aa62802b6ae31f43b47fc5e17ff1cb3b79b7065ec42dc1ee402f",
+			"0x11329c45c6c4691bcd77cd97b9ecd27fc9108d909877abbc5ce894f081e0a4cc",
+			"0x098a826b2a276c2cbf3c67b88caaedbb62316e37a570f5bdac257f9b901d355b",
+			"0x56b14ea493e2280d63db496606a64f44646e2565e52ef36590bfa40e9a5dba63",
+			"0x16b83703c9ffd53c3deaffab57d5d3c7252cebeec4a0b1ae957c17e56613a19b",
+			"0xfc004311f603989357c1bffde7a42a59227d0ae6ce052ef010f8c9f4c31926d3",
+			"0x1da81ae9ecb576ef9d74b05d15364e904cb252a54403fb2427eef5b49c8655fd",
+			"0x2260c1c27d3bac445984b282a3432361d68f4b2f61165cb330854d3b8507c4ce",
+			"0x0225d0c83107075de447a805d0892a8affec5e24b8229d79f327021a0848397c",
+			"0xf1fb6333242f439dc53aff6bc68cd267f21f77fbe805b824710412b2f45b5468",
+			"0x9fd2c82902d6890fad571d4125c02a1308dcc05ec53ace2b3a0c7d3cc46d16c1",
+			"0x7b9248e0a83b6cc1e47e4828ad33af3b37cbb586295727d9b19fdf2ef62ec4ba",
+			"0x352a0c2a06c5036b590a640c7d080a248dc1ddc1dbae67a5cf6698108b75955b",
+			"0x6737bb1c9b05f740e28f1813f57a8f9d18ebfd042f260edef963fafc553994ca",
+			"0x40d02212c95bfac0971766e71a339890ce86cfa96e2bd0ef0b270976062c5344",
+			"0xa493f9c86347f4b0c5d0d29232afdd5681e7b0582575ca88574a2e04d3dfd061",
+			"0xb748487ec35f299c2527260495fc2155ee354c488040f8b8de7aab0c232f6a5b",
+			"0x76ba5a1f7c4d314bc773e241dcd15e0237de6d5d1a7f990d1d750da9e4bfe2ea",
+			"0xcd8a379b7dff8c0732bfd151fb4c77b631e7d018b513cd34a2c7e1be3ed5b10f",
+			"0x29d471c421d1624ffe4093894d79aa57e818dcf4042cf58a10fe244f1d2b43b3",
+			"0x50a0d79e38620fb0f28dced7689486bc9790389041fc6b2731be8908c8f0027b",
+			"0x1587ad177b7eef5a475045374b69ab13a38101a92fa9bf3c2d7b686b5058b0f9",
+			"0xd79d13f96b5a91cde57290a87678630ce212aa11974f4244e2468f2a245796d6",
+			"0x8f86548742ea82892829ced29ecf75947b42743e25c4c0b485f66160e0a2c1fc",
+			"0x09650fb97435d4c7ae472f2b6cb3b9efcfde8f2c67e6c007c6c7b6f89529af16",
+			"0x60221758839c34e2841c76a443675df9934027e1d365c91136d9d7089fc8c373",
+			"0x4a524bd23dce7b76d06158ae7c8c8b973be2d8edafab5b6921fa1c2cba3cc4d9",
+			"0xd6e51f25afb4eb839766ffde785a598185dd1e34870538a4a8b4eb9115a045fd",
+			"0x84d6430ed5924bbc86da700f650536f3474f093bc9f61e52ee4f7f74a207464b",
+			"0x9d2cb8b8ba17dfb9fb7562f902a43219c977ffd38697b990bb11b3708ff3718a",
+			"0x89909eb848d74ee94b1b73ea6b1af8058ff771491ad131d3e13e2d95c2115903",
+			"0x118b3ae7ad25ab96fe8a63973312c758d4ce9ecd39cc24913c26a65b4b5534de",
+			"0x19cd088af8dbe2d3e6ca7987d9ee1564ea2256f482840b1d2f0da85060de9a86",
+			"0x98bc07422cf8b0c4d1428afb759300d9a7637de2518528d34f7d237be7e863be",
+			"0x0c64526b393066911c7da3f17f9e652cfa38112ae324e3c84416e811d3fe7cad"
 		]
 	},
 	"nodes": [
+		"enode://81863f47e9bd652585d3f78b4b2ee07b93dad603fd9bc3c293e1244250725998adc88da0cef48f1de89b15ab92b15db8f43dc2b6fb8fbd86a6f217a1dd886701@193.70.55.37:30303",
+		"enode://4afb3a9137a88267c02651052cf6fb217931b8c78ee058bb86643542a4e2e0a8d24d47d871654e1b78a276c363f3c1bc89254a973b00adc359c9e9a48f140686@144.217.139.5:30303",
+		"enode://c16d390b32e6eb1c312849fe12601412313165df1a705757d671296f1ac8783c5cff09eab0118ac1f981d7148c85072f0f26407e5c68598f3ad49209fade404d@139.99.51.203:30303",
+		"enode://4faf867a2e5e740f9b874e7c7355afee58a2d1ace79f7b692f1d553a1134eddbeb5f9210dd14dc1b774a46fd5f063a8bc1fa90579e13d9d18d1f59bac4a4b16b@139.99.160.213:30303",
 		"enode://6a868ced2dec399c53f730261173638a93a40214cf299ccf4d42a76e3fa54701db410669e8006347a4b3a74fa090bb35af0320e4bc8d04cf5b7f582b1db285f5@163.172.131.191:30303",
 		"enode://66a483383882a518fcc59db6c017f9cd13c71261f13c8d7e67ed43adbbc82a932d88d2291f59be577e9425181fc08828dc916fdd053af935a9491edf9d6006ba@212.47.247.103:30303",
 		"enode://cd6611461840543d5b9c56fbf088736154c699c43973b3a1a32390cf27106f87e58a818a606ccb05f3866de95a4fe860786fea71bf891ea95f234480d3022aa3@163.172.157.114:30303",
-		"enode://78b094cb27ceeecbe311bc278f4fde8b9a265db42d268c88484c94d7a2d19b82a1bd22dfd6c2bd4d90f9b05e6d42255e6eb85de15f73848ff82ed0be9cdf5202@52.233.198.218:30303",
-		"enode://00526537cb7e1aa6cf49714f0635fd0f608904d8d0693b949eea2dcdfdb0abbe4c794003a5fe57aa662d0a9215e8dfa4d2deb6ef0101c5e185e2617721813d43@40.65.122.44:30303",
-		"enode://4a456b4b6e6ee1f51389763e51b80fe04782c762445d96c32a96ebd34bd9178c1894924d5101123eacfd4f0fc4da25b5e1ee7f18832ac0bf4c6d6ac81442d698@40.71.6.49:3030",
-		"enode://68f85e7403976aa92318eff804cbe9bc988e0f5230d9d07ae4def030cbae16603262638e272d19875b7e5c54e296ba88ab6ec6e98face9e2537346c4dce78882@52.243.47.211:30303",
-		"enode://dc72806c3aa8fda207c8c018aba8d6cf143728b3628b6ded8d5e8cdeb8aa05cbd53f710ecd014c9a8f0d1e98f2874bff8afb15a229202f510a9c0258d1f6d109@159.203.210.80:30303",
-		"enode://5a62f19d35c0da8b576c9414568c728d4744e6e9d436c0f9db27456400011414f515871f13a6b8e0468534b5116cfe765d7630f680f1707a38467940a9f62511@45.55.33.62:30303",
-		"enode://605e04a43b1156966b3a3b66b980c87b7f18522f7f712035f84576016be909a2798a438b2b17b1a8c58db314d88539a77419ca4be36148c086900fba487c9d39@188.166.255.12:30303",
 		"enode://1d1f7bcb159d308eb2f3d5e32dc5f8786d714ec696bb2f7e3d982f9bcd04c938c139432f13aadcaf5128304a8005e8606aebf5eebd9ec192a1471c13b5e31d49@138.201.223.35:30303",
 		"enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303",
 		"enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303",
@@ -2782,7 +3036,6 @@
 		"enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30308",
 		"enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30309",
 		"enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303",
-		"enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303",
 		"enode://4cd540b2c3292e17cff39922e864094bf8b0741fcc8c5dcea14957e389d7944c70278d872902e3d0345927f621547efa659013c400865485ab4bfa0c6596936f@138.201.144.135:30303",
 		"enode://01f76fa0561eca2b9a7e224378dd854278735f1449793c46ad0c4e79e8775d080c21dcc455be391e90a98153c3b05dcc8935c8440de7b56fe6d67251e33f4e3c@51.15.42.252:30303",
 		"enode://2c9059f05c352b29d559192fe6bca272d965c9f2290632a2cfda7f83da7d2634f3ec45ae3a72c54dd4204926fb8082dcf9686e0d7504257541c86fc8569bcf4b@163.172.171.38:30303",

ethcore/service/Cargo.toml

@@ -13,6 +13,7 @@ ethcore-sync = { path = "../sync" }
 kvdb = { path = "../../util/kvdb" }
 log = "0.3"
 stop-guard = { path = "../../util/stop-guard" }
+trace-time = { path = "../../util/trace-time" }
 
 [dev-dependencies]
 tempdir = "0.3"

ethcore/service/src/lib.rs

@@ -28,6 +28,9 @@ extern crate error_chain;
 #[macro_use]
 extern crate log;
 
+#[macro_use]
+extern crate trace_time;
+
 #[cfg(test)]
 extern crate tempdir;
 

ethcore/service/src/service.rs

@@ -29,11 +29,11 @@ use sync::PrivateTxHandler;
 use ethcore::client::{Client, ClientConfig, ChainNotify, ClientIoMessage};
 use ethcore::miner::Miner;
 use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams};
-use ethcore::snapshot::{RestorationStatus};
+use ethcore::snapshot::{SnapshotService as _SnapshotService, RestorationStatus};
 use ethcore::spec::Spec;
 use ethcore::account_provider::AccountProvider;
 
-use ethcore_private_tx;
+use ethcore_private_tx::{self, Importer};
 use Error;
 
 pub struct PrivateTxService {
@@ -112,14 +112,13 @@ impl ClientService {
 				account_provider,
 				encryptor,
 				private_tx_conf,
-				io_service.channel())?,
-		);
+				io_service.channel(),
+		));
 		let private_tx = Arc::new(PrivateTxService::new(provider));
 
 		let client_io = Arc::new(ClientIoHandler {
 			client: client.clone(),
 			snapshot: snapshot.clone(),
-			private_tx: private_tx.clone(),
 		});
 		io_service.register_handler(client_io)?;
 
@@ -169,13 +168,17 @@ impl ClientService {
 
 	/// Get a handle to the database.
 	pub fn db(&self) -> Arc<KeyValueDB> { self.database.clone() }
+
+	/// Shutdown the Client Service
+	pub fn shutdown(&self) {
+		self.snapshot.shutdown();
+	}
 }
 
 /// IO interface for the Client handler
 struct ClientIoHandler {
 	client: Arc<Client>,
 	snapshot: Arc<SnapshotService>,
-	private_tx: Arc<PrivateTxService>,
 }
 
 const CLIENT_TICK_TIMER: TimerToken = 0;
@@ -191,6 +194,7 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
 	}
 
 	fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
+		trace_time!("service::read");
 		match timer {
 			CLIENT_TICK_TIMER => {
 				use ethcore::snapshot::SnapshotService;
@@ -203,20 +207,24 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
 	}
 
 	fn message(&self, _io: &IoContext<ClientIoMessage>, net_message: &ClientIoMessage) {
+		trace_time!("service::message");
 		use std::thread;
 
 		match *net_message {
-			ClientIoMessage::BlockVerified => { self.client.import_verified_blocks(); }
-			ClientIoMessage::NewTransactions(ref transactions, peer_id) => {
-				self.client.import_queued_transactions(transactions, peer_id);
+			ClientIoMessage::BlockVerified => {
+				self.client.import_verified_blocks();
 			}
 			ClientIoMessage::BeginRestoration(ref manifest) => {
 				if let Err(e) = self.snapshot.init_restore(manifest.clone(), true) {
 					warn!("Failed to initialize snapshot restoration: {}", e);
 				}
 			}
-			ClientIoMessage::FeedStateChunk(ref hash, ref chunk) => self.snapshot.feed_state_chunk(*hash, chunk),
-			ClientIoMessage::FeedBlockChunk(ref hash, ref chunk) => self.snapshot.feed_block_chunk(*hash, chunk),
+			ClientIoMessage::FeedStateChunk(ref hash, ref chunk) => {
+				self.snapshot.feed_state_chunk(*hash, chunk)
+			}
+			ClientIoMessage::FeedBlockChunk(ref hash, ref chunk) => {
+				self.snapshot.feed_block_chunk(*hash, chunk)
+			}
 			ClientIoMessage::TakeSnapshot(num) => {
 				let client = self.client.clone();
 				let snapshot = self.snapshot.clone();
@@ -231,12 +239,9 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
 					debug!(target: "snapshot", "Failed to initialize periodic snapshot thread: {:?}", e);
 				}
 			},
-			ClientIoMessage::NewMessage(ref message) => if let Err(e) = self.client.engine().handle_message(message) {
-				trace!(target: "poa", "Invalid message received: {}", e);
-			},
-			ClientIoMessage::NewPrivateTransaction => if let Err(e) = self.private_tx.provider.on_private_transaction_queued() {
-				warn!("Failed to handle private transaction {:?}", e);
-			},
+			ClientIoMessage::Execute(ref exec) => {
+				(*exec.0)(&self.client);
+			}
 			_ => {} // ignore other messages
 		}
 	}

ethcore/src/account_provider/mod.rs

@@ -272,8 +272,8 @@ impl AccountProvider {
 	}
 
 	/// Checks whether an account with a given address is present.
-	pub fn has_account(&self, address: Address) -> Result<bool, Error> {
-		Ok(self.sstore.account_ref(&address).is_ok() && !self.blacklisted_accounts.contains(&address))
+	pub fn has_account(&self, address: Address) -> bool {
+		self.sstore.account_ref(&address).is_ok() && !self.blacklisted_accounts.contains(&address)
 	}
 
 	/// Returns addresses of all accounts.

ethcore/src/blockchain/blockchain.rs

@@ -57,6 +57,12 @@ pub trait BlockProvider {
 	/// (though not necessarily a part of the canon chain).
 	fn is_known(&self, hash: &H256) -> bool;
 
+	/// Returns true if the given block is known and in the canon chain.
+	fn is_canon(&self, hash: &H256) -> bool {
+		let is_canon = || Some(hash == &self.block_hash(self.block_number(hash)?)?);
+		is_canon().unwrap_or(false)
+	}
+
 	/// Get the first block of the best part of the chain.
 	/// Return `None` if there is no gap and the first block is the genesis.
 	/// Any queries of blocks which precede this one are not guaranteed to
@@ -148,7 +154,7 @@ pub trait BlockProvider {
 	fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
 
 	/// Returns logs matching given filter.
-	fn logs<F>(&self, blocks: Vec<BlockNumber>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
+	fn logs<F>(&self, blocks: Vec<H256>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
 		where F: Fn(&LogEntry) -> bool + Send + Sync, Self: Sized;
 }
 
@@ -332,16 +338,18 @@ impl BlockProvider for BlockChain {
 			.collect()
 	}
 
-	fn logs<F>(&self, mut blocks: Vec<BlockNumber>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
+	/// Returns logs matching given filter. The order of logs returned will be the same as the order of the blocks
+	/// provided. And it's the callers responsibility to sort blocks provided in advance.
+	fn logs<F>(&self, mut blocks: Vec<H256>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
 		where F: Fn(&LogEntry) -> bool + Send + Sync, Self: Sized {
 		// sort in reverse order
-		blocks.sort_by(|a, b| b.cmp(a));
+		blocks.reverse();
 
 		let mut logs = blocks
 			.chunks(128)
 			.flat_map(move |blocks_chunk| {
 				blocks_chunk.into_par_iter()
-					.filter_map(|number| self.block_hash(*number).map(|hash| (*number, hash)))
+					.filter_map(|hash| self.block_number(&hash).map(|r| (r, hash)))
 					.filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
 					.filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, b.transaction_hashes())))
 					.flat_map(|(number, hash, mut receipts, mut hashes)| {
@@ -368,7 +376,7 @@ impl BlockProvider for BlockChain {
 									.enumerate()
 									.map(move |(i, log)| LocalizedLogEntry {
 										entry: log,
-										block_hash: hash,
+										block_hash: *hash,
 										block_number: number,
 										transaction_hash: tx_hash,
 										// iterating in reverse order
@@ -430,7 +438,7 @@ impl<'a> Iterator for EpochTransitionIter<'a> {
 				return None
 			}
 
-			let transitions: EpochTransitions = ::rlp::decode(&val[..]);
+			let transitions: EpochTransitions = ::rlp::decode(&val[..]).expect("decode error: the db is corrupted or the data structure has changed");
 
 			// if there are multiple candidates, at most one will be on the
 			// canon chain.
@@ -454,7 +462,7 @@ impl<'a> Iterator for EpochTransitionIter<'a> {
 impl BlockChain {
 	/// Create new instance of blockchain from given Genesis.
 	pub fn new(config: Config, genesis: &[u8], db: Arc<KeyValueDB>) -> BlockChain {
-		// 400 is the avarage size of the key
+		// 400 is the average size of the key
 		let cache_man = CacheManager::new(config.pref_cache_size, config.max_cache_size, 400);
 
 		let mut bc = BlockChain {
@@ -1386,18 +1394,24 @@ impl BlockChain {
 
 	/// Returns general blockchain information
 	pub fn chain_info(&self) -> BlockChainInfo {
+		// Make sure to call internal methods first to avoid
+		// recursive locking of `best_block`.
+		let first_block_hash = self.first_block();
+		let first_block_number = self.first_block_number().into();
+		let genesis_hash = self.genesis_hash();
+
 		// ensure data consistencly by locking everything first
 		let best_block = self.best_block.read();
 		let best_ancient_block = self.best_ancient_block.read();
 		BlockChainInfo {
 			total_difficulty: best_block.total_difficulty,
 			pending_total_difficulty: best_block.total_difficulty,
-			genesis_hash: self.genesis_hash(),
+			genesis_hash,
 			best_block_hash: best_block.header.hash(),
 			best_block_number: best_block.header.number(),
 			best_block_timestamp: best_block.header.timestamp(),
-			first_block_hash: self.first_block(),
-			first_block_number: From::from(self.first_block_number()),
+			first_block_hash,
+			first_block_number,
 			ancient_block_hash: best_ancient_block.as_ref().map(|b| b.hash),
 			ancient_block_number: best_ancient_block.as_ref().map(|b| b.number),
 		}
@@ -1933,17 +1947,33 @@ mod tests {
 			value: 103.into(),
 			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
 		}.sign(&secret(), None);
+		let t4 = Transaction {
+			nonce: 0.into(),
+			gas_price: 0.into(),
+			gas: 100_000.into(),
+			action: Action::Create,
+			value: 104.into(),
+			data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(),
+		}.sign(&secret(), None);
 		let tx_hash1 = t1.hash();
 		let tx_hash2 = t2.hash();
 		let tx_hash3 = t3.hash();
+		let tx_hash4 = t4.hash();
 
 		let genesis = BlockBuilder::genesis();
 		let b1 = genesis.add_block_with_transactions(vec![t1, t2]);
 		let b2 = b1.add_block_with_transactions(iter::once(t3));
+		let b3 = genesis.add_block_with(|| BlockOptions {
+			transactions: vec![t4.clone()],
+			difficulty: U256::from(9),
+			..Default::default()
+		}); // Branch block
 		let b1_hash = b1.last().hash();
 		let b1_number = b1.last().number();
 		let b2_hash = b2.last().hash();
 		let b2_number = b2.last().number();
+		let b3_hash = b3.last().hash();
+		let b3_number = b3.last().number();
 
 		let db = new_db();
 		let bc = new_chain(&genesis.last().encoded(), db.clone());
@@ -1974,10 +2004,21 @@ mod tests {
 				],
 			}
 		]);
+		insert_block(&db, &bc, &b3.last().encoded(), vec![
+			Receipt {
+				outcome: TransactionOutcome::StateRoot(H256::default()),
+				gas_used: 10_000.into(),
+				log_bloom: Default::default(),
+				logs: vec![
+					LogEntry { address: Default::default(), topics: vec![], data: vec![5], },
+				],
+			}
+		]);
 
 		// when
-		let logs1 = bc.logs(vec![1, 2], |_| true, None);
-		let logs2 = bc.logs(vec![1, 2], |_| true, Some(1));
+		let logs1 = bc.logs(vec![b1_hash, b2_hash], |_| true, None);
+		let logs2 = bc.logs(vec![b1_hash, b2_hash], |_| true, Some(1));
+		let logs3 = bc.logs(vec![b3_hash], |_| true, None);
 
 		// then
 		assert_eq!(logs1, vec![
@@ -2029,6 +2070,17 @@ mod tests {
 				log_index: 0,
 			}
 		]);
+		assert_eq!(logs3, vec![
+			LocalizedLogEntry {
+				entry: LogEntry { address: Default::default(), topics: vec![], data: vec![5] },
+				block_hash: b3_hash,
+				block_number: b3_number,
+				transaction_hash: tx_hash4,
+				transaction_index: 0,
+				transaction_log_index: 0,
+				log_index: 0,
+			}
+		]);
 	}
 
 	#[test]

ethcore/src/blockchain/import_route.rs

@@ -20,7 +20,7 @@ use ethereum_types::H256;
 use blockchain::block_info::{BlockInfo, BlockLocation};
 
 /// Import route for newly inserted block.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct ImportRoute {
 	/// Blocks that were invalidated by new block.
 	pub retracted: Vec<H256>,

ethcore/src/client/ancient_import.rs

@@ -32,16 +32,16 @@ const HEAVY_VERIFY_RATE: f32 = 0.02;
 /// Ancient block verifier: import an ancient sequence of blocks in order from a starting
 /// epoch.
 pub struct AncientVerifier {
-	cur_verifier: RwLock<Box<EpochVerifier<EthereumMachine>>>,
+	cur_verifier: RwLock<Option<Box<EpochVerifier<EthereumMachine>>>>,
 	engine: Arc<EthEngine>,
 }
 
 impl AncientVerifier {
-	/// Create a new ancient block verifier with the given engine and initial verifier.
-	pub fn new(engine: Arc<EthEngine>, start_verifier: Box<EpochVerifier<EthereumMachine>>) -> Self {
+	/// Create a new ancient block verifier with the given engine.
+	pub fn new(engine: Arc<EthEngine>) -> Self {
 		AncientVerifier {
-			cur_verifier: RwLock::new(start_verifier),
-			engine: engine,
+			cur_verifier: RwLock::new(None),
+			engine,
 		}
 	}
 
@@ -53,17 +53,49 @@ impl AncientVerifier {
 		header: &Header,
 		chain: &BlockChain,
 	) -> Result<(), ::error::Error> {
-		match rng.gen::<f32>() <= HEAVY_VERIFY_RATE {
-			true => self.cur_verifier.read().verify_heavy(header)?,
-			false => self.cur_verifier.read().verify_light(header)?,
+		// perform verification
+		let verified = if let Some(ref cur_verifier) = *self.cur_verifier.read() {
+			match rng.gen::<f32>() <= HEAVY_VERIFY_RATE {
+				true => cur_verifier.verify_heavy(header)?,
+				false => cur_verifier.verify_light(header)?,
+			}
+			true
+		} else {
+			false
+		};
+
+		// when there is no verifier initialize it.
+		// We use a bool flag to avoid double locking in the happy case
+		if !verified {
+			{
+				let mut cur_verifier = self.cur_verifier.write();
+				if cur_verifier.is_none() {
+					*cur_verifier = Some(self.initial_verifier(header, chain)?);
+				}
+			}
+			// Call again to verify.
+			return self.verify(rng, header, chain);
 		}
 
 		// ancient import will only use transitions obtained from the snapshot.
 		if let Some(transition) = chain.epoch_transition(header.number(), header.hash()) {
 			let v = self.engine.epoch_verifier(&header, &transition.proof).known_confirmed()?;
-			*self.cur_verifier.write() = v;
+			*self.cur_verifier.write() = Some(v);
 		}
 
 		Ok(())
 	}
+
+	fn initial_verifier(&self, header: &Header, chain: &BlockChain)
+		-> Result<Box<EpochVerifier<EthereumMachine>>, ::error::Error>
+	{
+		trace!(target: "client", "Initializing ancient block restoration.");
+		let current_epoch_data = chain.epoch_transitions()
+			.take_while(|&(_, ref t)| t.block_number < header.number())
+			.last()
+			.map(|(_, t)| t.proof)
+			.expect("At least one epoch entry (genesis) always stored; qed");
+
+		self.engine.epoch_verifier(&header, &current_epoch_data).known_confirmed()
+	}
 }

ethcore/src/client/chain_notify.rs

@@ -17,6 +17,9 @@
 use bytes::Bytes;
 use ethereum_types::H256;
 use transaction::UnverifiedTransaction;
+use blockchain::ImportRoute;
+use std::time::Duration;
+use std::collections::HashMap;
 
 /// Messages to broadcast via chain
 pub enum ChainMessageType {
@@ -28,6 +31,89 @@ pub enum ChainMessageType {
 	SignedPrivateTransaction(Vec<u8>),
 }
 
+/// Route type to indicate whether it is enacted or retracted.
+#[derive(Clone)]
+pub enum ChainRouteType {
+	/// Enacted block
+	Enacted,
+	/// Retracted block
+	Retracted
+}
+
+/// A complete chain enacted retracted route.
+#[derive(Default, Clone)]
+pub struct ChainRoute {
+	route: Vec<(H256, ChainRouteType)>,
+	enacted: Vec<H256>,
+	retracted: Vec<H256>,
+}
+
+impl<'a> From<&'a [ImportRoute]> for ChainRoute {
+	fn from(import_results: &'a [ImportRoute]) -> ChainRoute {
+		ChainRoute::new(import_results.iter().flat_map(|route| {
+			route.retracted.iter().map(|h| (*h, ChainRouteType::Retracted))
+				.chain(route.enacted.iter().map(|h| (*h, ChainRouteType::Enacted)))
+		}).collect())
+	}
+}
+
+impl ChainRoute {
+	/// Create a new ChainRoute based on block hash and route type pairs.
+	pub fn new(route: Vec<(H256, ChainRouteType)>) -> Self {
+		let (enacted, retracted) = Self::to_enacted_retracted(&route);
+
+		Self { route, enacted, retracted }
+	}
+
+	/// Gather all non-duplicate enacted and retracted blocks.
+	fn to_enacted_retracted(route: &[(H256, ChainRouteType)]) -> (Vec<H256>, Vec<H256>) {
+		fn map_to_vec(map: Vec<(H256, bool)>) -> Vec<H256> {
+			map.into_iter().map(|(k, _v)| k).collect()
+		}
+
+		// Because we are doing multiple inserts some of the blocks that were enacted in import `k`
+		// could be retracted in import `k+1`. This is why to understand if after all inserts
+		// the block is enacted or retracted we iterate over all routes and at the end final state
+		// will be in the hashmap
+		let map = route.iter().fold(HashMap::new(), |mut map, route| {
+			match &route.1 {
+				&ChainRouteType::Enacted => {
+					map.insert(route.0, true);
+				},
+				&ChainRouteType::Retracted => {
+					map.insert(route.0, false);
+				},
+			}
+			map
+		});
+
+		// Split to enacted retracted (using hashmap value)
+		let (enacted, retracted) = map.into_iter().partition(|&(_k, v)| v);
+		// And convert tuples to keys
+		(map_to_vec(enacted), map_to_vec(retracted))
+	}
+
+	/// Consume route and return the enacted retracted form.
+	pub fn into_enacted_retracted(self) -> (Vec<H256>, Vec<H256>) {
+		(self.enacted, self.retracted)
+	}
+
+	/// All non-duplicate enacted blocks.
+	pub fn enacted(&self) -> &[H256] {
+		&self.enacted
+	}
+
+	/// All non-duplicate retracted blocks.
+	pub fn retracted(&self) -> &[H256] {
+		&self.retracted
+	}
+
+	/// All blocks in the route.
+	pub fn route(&self) -> &[(H256, ChainRouteType)] {
+		&self.route
+	}
+}
+
 /// Represents what has to be handled by actor listening to chain events
 pub trait ChainNotify : Send + Sync {
 	/// fires when chain has new blocks.
@@ -35,12 +121,11 @@ pub trait ChainNotify : Send + Sync {
 		&self,
 		_imported: Vec<H256>,
 		_invalid: Vec<H256>,
-		_enacted: Vec<H256>,
-		_retracted: Vec<H256>,
+		_route: ChainRoute,
 		_sealed: Vec<H256>,
 		// Block bytes.
 		_proposed: Vec<Bytes>,
-		_duration: u64,
+		_duration: Duration,
 	) {
 		// does nothing by default
 	}

ethcore/src/client/client.rs

@@ -14,16 +14,17 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::collections::{HashSet, HashMap, BTreeMap, VecDeque};
+use std::collections::{HashSet, BTreeMap, BTreeSet, VecDeque};
+use std::fmt;
 use std::str::FromStr;
 use std::sync::{Arc, Weak};
 use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
-use std::time::{Instant};
-use itertools::Itertools;
+use std::time::{Instant, Duration};
 
 // util
 use hash::keccak;
 use bytes::Bytes;
+use itertools::Itertools;
 use journaldb;
 use trie::{TrieSpec, TrieFactory, Trie};
 use kvdb::{DBValue, KeyValueDB, DBTransaction};
@@ -32,7 +33,7 @@ use util_error::UtilError;
 // other
 use ethereum_types::{H256, Address, U256};
 use block::{IsBlock, LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock};
-use blockchain::{BlockChain, BlockProvider,  TreeRoute, ImportRoute, TransactionAddress};
+use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute, TransactionAddress};
 use client::ancient_import::AncientVerifier;
 use client::Error as ClientError;
 use client::{
@@ -45,7 +46,8 @@ use client::{
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
 	TraceFilter, CallAnalytics, BlockImportError, Mode,
-	ChainNotify, PruningInfo, ProvingBlockChainClient, EngineInfo, ChainMessageType
+	ChainMessageType, ChainNotify, ChainRoute, PruningInfo, ProvingBlockChainClient,
+	EngineInfo, IoClient,
 };
 use encoded;
 use engines::{EthEngine, EpochTransition};
@@ -55,14 +57,13 @@ use evm::Schedule;
 use executive::{Executive, Executed, TransactOptions, contract_address};
 use factory::{Factories, VmFactory};
 use header::{BlockNumber, Header};
-use io::IoChannel;
+use io::{IoChannel, IoError};
 use log_entry::LocalizedLogEntry;
 use miner::{Miner, MinerService};
 use ethcore_miner::pool::VerifiedTransaction;
 use parking_lot::{Mutex, RwLock};
 use rand::OsRng;
 use receipt::{Receipt, LocalizedReceipt};
-use rlp::Rlp;
 use snapshot::{self, io as snapshot_io};
 use spec::Spec;
 use state_db::StateDB;
@@ -86,6 +87,9 @@ pub use verification::queue::QueueInfo as BlockQueueInfo;
 use_contract!(registry, "Registry", "res/contracts/registrar.json");
 
 const MAX_TX_QUEUE_SIZE: usize = 4096;
+const MAX_ANCIENT_BLOCKS_QUEUE_SIZE: usize = 4096;
+// Max number of blocks imported at once.
+const MAX_ANCIENT_BLOCKS_TO_IMPORT: usize = 4;
 const MAX_QUEUE_SIZE_TO_SLEEP_ON: usize = 2;
 const MIN_HISTORY_SIZE: u64 = 8;
 
@@ -121,7 +125,7 @@ impl<'a> ::std::ops::Sub<&'a ClientReport> for ClientReport {
 		self.blocks_imported -= other.blocks_imported;
 		self.transactions_applied -= other.transactions_applied;
 		self.gas_processed = self.gas_processed - other.gas_processed;
-		self.state_db_mem  = higher_mem - lower_mem;
+		self.state_db_mem = higher_mem - lower_mem;
 
 		self
 	}
@@ -155,10 +159,7 @@ struct Importer {
 	pub miner: Arc<Miner>,
 
 	/// Ancient block verifier: import an ancient sequence of blocks in order from a starting epoch
-	pub ancient_verifier: Mutex<Option<AncientVerifier>>,
-
-	/// Random number generator used by `AncientVerifier`
-	pub rng: Mutex<OsRng>,
+	pub ancient_verifier: AncientVerifier,
 
 	/// Ethereum engine to be used during import
 	pub engine: Arc<EthEngine>,
@@ -205,8 +206,19 @@ pub struct Client {
 	/// List of actors to be notified on certain chain events
 	notify: RwLock<Vec<Weak<ChainNotify>>>,
 
-	/// Count of pending transactions in the queue
-	queue_transactions: AtomicUsize,
+	/// Queued transactions from IO
+	queue_transactions: IoChannelQueue,
+	/// Ancient blocks import queue
+	queue_ancient_blocks: IoChannelQueue,
+	/// Queued ancient blocks, make sure they are imported in order.
+	queued_ancient_blocks: Arc<RwLock<(
+		HashSet<H256>,
+		VecDeque<(Header, Bytes, Bytes)>
+	)>>,
+	ancient_blocks_import_lock: Arc<Mutex<()>>,
+	/// Consensus messages import queue
+	queue_consensus_message: IoChannelQueue,
+
 	last_hashes: RwLock<VecDeque<H256>>,
 	factories: Factories,
 
@@ -240,38 +252,11 @@ impl Importer {
 			verifier: verification::new(config.verifier_type.clone()),
 			block_queue,
 			miner,
-			ancient_verifier: Mutex::new(None),
-			rng: Mutex::new(OsRng::new()?),
+			ancient_verifier: AncientVerifier::new(engine.clone()),
 			engine,
 		})
 	}
 
-	fn calculate_enacted_retracted(&self, import_results: &[ImportRoute]) -> (Vec<H256>, Vec<H256>) {
-		fn map_to_vec(map: Vec<(H256, bool)>) -> Vec<H256> {
-			map.into_iter().map(|(k, _v)| k).collect()
-		}
-
-		// In ImportRoute we get all the blocks that have been enacted and retracted by single insert.
-		// Because we are doing multiple inserts some of the blocks that were enacted in import `k`
-		// could be retracted in import `k+1`. This is why to understand if after all inserts
-		// the block is enacted or retracted we iterate over all routes and at the end final state
-		// will be in the hashmap
-		let map = import_results.iter().fold(HashMap::new(), |mut map, route| {
-			for hash in &route.enacted {
-				map.insert(hash.clone(), true);
-			}
-			for hash in &route.retracted {
-				map.insert(hash.clone(), false);
-			}
-			map
-		});
-
-		// Split to enacted retracted (using hashmap value)
-		let (enacted, retracted) = map.into_iter().partition(|&(_k, v)| v);
-		// And convert tuples to keys
-		(map_to_vec(enacted), map_to_vec(retracted))
-	}
-
 	/// This is triggered by a message coming from a block queue when the block is ready for insertion
 	pub fn import_verified_blocks(&self, client: &Client) -> usize {
 
@@ -332,27 +317,22 @@ impl Importer {
 				self.block_queue.mark_as_bad(&invalid_blocks);
 			}
 			let is_empty = self.block_queue.mark_as_good(&imported_blocks);
-			let duration_ns = {
-				let elapsed = start.elapsed();
-				elapsed.as_secs() * 1_000_000_000 + elapsed.subsec_nanos() as u64
-			};
-			(imported_blocks, import_results, invalid_blocks, imported, proposed_blocks, duration_ns, is_empty)
+			(imported_blocks, import_results, invalid_blocks, imported, proposed_blocks, start.elapsed(), is_empty)
 		};
 
 		{
 			if !imported_blocks.is_empty() && is_empty {
-				let (enacted, retracted) = self.calculate_enacted_retracted(&import_results);
+				let route = ChainRoute::from(import_results.as_ref());
 
 				if is_empty {
-					self.miner.chain_new_blocks(client, &imported_blocks, &invalid_blocks, &enacted, &retracted, false);
+					self.miner.chain_new_blocks(client, &imported_blocks, &invalid_blocks, route.enacted(), route.retracted(), false);
 				}
 
 				client.notify(|notify| {
 					notify.new_blocks(
 						imported_blocks.clone(),
 						invalid_blocks.clone(),
-						enacted.clone(),
-						retracted.clone(),
+						route.clone(),
 						Vec::new(),
 						proposed_blocks.clone(),
 						duration,
@@ -448,57 +428,25 @@ impl Importer {
 		Ok(locked_block)
 	}
 
+
 	/// Import a block with transaction receipts.
 	///
 	/// The block is guaranteed to be the next best blocks in the
 	/// first block sequence. Does no sealing or transaction validation.
-	fn import_old_block(&self, block_bytes: Bytes, receipts_bytes: Bytes, db: &KeyValueDB, chain: &BlockChain) -> Result<H256, ::error::Error> {
-		let block = view!(BlockView, &block_bytes);
-		let header = block.header();
-		let receipts = ::rlp::decode_list(&receipts_bytes);
+	fn import_old_block(&self, header: &Header, block_bytes: &[u8], receipts_bytes: &[u8], db: &KeyValueDB, chain: &BlockChain) -> Result<H256, ::error::Error> {
+		let receipts = ::rlp::decode_list(receipts_bytes);
 		let hash = header.hash();
 		let _import_lock = self.import_lock.lock();
 
 		{
 			trace_time!("import_old_block");
-			let mut ancient_verifier = self.ancient_verifier.lock();
-
-			{
-				// closure for verifying a block.
-				let verify_with = |verifier: &AncientVerifier| -> Result<(), ::error::Error> {
-					// verify the block, passing the chain for updating the epoch
-					// verifier.
-					let mut rng = OsRng::new().map_err(UtilError::from)?;
-					verifier.verify(&mut rng, &header, &chain)
-				};
-
-				// initialize the ancient block verifier if we don't have one already.
-				match &mut *ancient_verifier {
-					&mut Some(ref verifier) => {
-						verify_with(verifier)?
-					}
-					x @ &mut None => {
-						// load most recent epoch.
-						trace!(target: "client", "Initializing ancient block restoration.");
-						let current_epoch_data = chain.epoch_transitions()
-							.take_while(|&(_, ref t)| t.block_number < header.number())
-							.last()
-							.map(|(_, t)| t.proof)
-							.expect("At least one epoch entry (genesis) always stored; qed");
-
-						let current_verifier = self.engine.epoch_verifier(&header, &current_epoch_data)
-							.known_confirmed()?;
-						let current_verifier = AncientVerifier::new(self.engine.clone(), current_verifier);
-
-						verify_with(&current_verifier)?;
-						*x = Some(current_verifier);
-					}
-				}
-			}
+			// verify the block, passing the chain for updating the epoch verifier.
+			let mut rng = OsRng::new().map_err(UtilError::from)?;
+			self.ancient_verifier.verify(&mut rng, &header, &chain)?;
 
 			// Commit results
 			let mut batch = DBTransaction::new();
-			chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, false, true);
+			chain.insert_unordered_block(&mut batch, block_bytes, receipts, None, false, true);
 			// Final commit to the DB
 			db.write_buffered(batch);
 			chain.commit();
@@ -768,7 +716,11 @@ impl Client {
 			report: RwLock::new(Default::default()),
 			io_channel: Mutex::new(message_channel),
 			notify: RwLock::new(Vec::new()),
-			queue_transactions: AtomicUsize::new(0),
+			queue_transactions: IoChannelQueue::new(MAX_TX_QUEUE_SIZE),
+			queue_ancient_blocks: IoChannelQueue::new(MAX_ANCIENT_BLOCKS_QUEUE_SIZE),
+			queued_ancient_blocks: Default::default(),
+			ancient_blocks_import_lock: Default::default(),
+			queue_consensus_message: IoChannelQueue::new(usize::max_value()),
 			last_hashes: RwLock::new(VecDeque::new()),
 			factories: factories,
 			history: history,
@@ -854,7 +806,7 @@ impl Client {
 	}
 
 	fn notify<F>(&self, f: F) where F: Fn(&ChainNotify) {
-		for np in self.notify.read().iter() {
+		for np in &*self.notify.read() {
 			if let Some(n) = np.upgrade() {
 				f(&*n);
 			}
@@ -988,24 +940,6 @@ impl Client {
 		}
 	}
 
-	/// Import transactions from the IO queue
-	pub fn import_queued_transactions(&self, transactions: &[Bytes], peer_id: usize) -> usize {
-		trace_time!("import_queued_transactions");
-		self.queue_transactions.fetch_sub(transactions.len(), AtomicOrdering::SeqCst);
-
-		let txs: Vec<UnverifiedTransaction> = transactions
-			.iter()
-			.filter_map(|bytes| Rlp::new(bytes).as_val().ok())
-			.collect();
-
-		self.notify(|notify| {
-			notify.transactions_received(&txs, peer_id);
-		});
-
-		let results = self.importer.miner.import_external_transactions(self, txs);
-		results.len()
-	}
-
 	/// Get shared miner reference.
 	#[cfg(test)]
 	pub fn miner(&self) -> Arc<Miner> {
@@ -1065,7 +999,8 @@ impl Client {
 	/// Otherwise, this can fail (but may not) if the DB prunes state.
 	pub fn state_at_beginning(&self, id: BlockId) -> Option<State<StateDB>> {
 		match self.block_number(id) {
-			None | Some(0) => None,
+			None => None,
+			Some(0) => self.state_at(id),
 			Some(n) => self.state_at(BlockId::Number(n - 1)),
 		}
 	}
@@ -1295,8 +1230,7 @@ impl Client {
 				=> Some(self.chain.read().best_block_header()),
 			BlockId::Number(number) if number == self.chain.read().best_block_number()
 				=> Some(self.chain.read().best_block_header()),
-			_
-				=> self.block_header(id).map(|h| h.decode()),
+			_   => self.block_header(id).and_then(|h| h.decode().ok())
 		}
 	}
 }
@@ -1413,7 +1347,7 @@ impl ImportBlock for Client {
 		use verification::queue::kind::blocks::Unverified;
 
 		// create unverified block here so the `keccak` calculation can be cached.
-		let unverified = Unverified::new(bytes);
+		let unverified = Unverified::from_rlp(bytes)?;
 
 		{
 			if self.chain.read().is_known(&unverified.hash()) {
@@ -1426,22 +1360,6 @@ impl ImportBlock for Client {
 		}
 		Ok(self.importer.block_queue.import(unverified)?)
 	}
-
-	fn import_block_with_receipts(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, BlockImportError> {
-		{
-			// check block order
-			let header = view!(BlockView, &block_bytes).header_view();
-			if self.chain.read().is_known(&header.hash()) {
-				bail!(BlockImportErrorKind::Import(ImportErrorKind::AlreadyInChain));
-			}
-			let status = self.block_status(BlockId::Hash(header.parent_hash()));
-			if  status == BlockStatus::Unknown || status == BlockStatus::Pending {
-				bail!(BlockImportErrorKind::Block(BlockError::UnknownParent(header.parent_hash())));
-			}
-		}
-
-		self.importer.import_old_block(block_bytes, receipts_bytes, &**self.db.read(), &*self.chain.read()).map_err(Into::into)
-	}
 }
 
 impl StateClient for Client {
@@ -1498,7 +1416,7 @@ impl Call for Client {
 	}
 
 	fn estimate_gas(&self, t: &SignedTransaction, state: &Self::State, header: &Header) -> Result<U256, CallError> {
-		let (mut upper, max_upper, env_info)  = {
+		let (mut upper, max_upper, env_info) = {
 			let init = *header.gas_limit();
 			let max = init * U256::from(10);
 
@@ -1853,23 +1771,82 @@ impl BlockChainClient for Client {
 	}
 
 	fn logs(&self, filter: Filter) -> Vec<LocalizedLogEntry> {
-		let (from, to) = match (self.block_number_ref(&filter.from_block), self.block_number_ref(&filter.to_block)) {
-			(Some(from), Some(to)) => (from, to),
-			_ => return Vec::new(),
+		// Wrap the logic inside a closure so that we can take advantage of question mark syntax.
+		let fetch_logs = || {
+			let chain = self.chain.read();
+
+			// First, check whether `filter.from_block` and `filter.to_block` is on the canon chain. If so, we can use the
+			// optimized version.
+			let is_canon = |id| {
+				match id {
+					// If it is referred by number, then it is always on the canon chain.
+					&BlockId::Earliest | &BlockId::Latest | &BlockId::Number(_) => true,
+					// If it is referred by hash, we see whether a hash -> number -> hash conversion gives us the same
+					// result.
+					&BlockId::Hash(ref hash) => chain.is_canon(hash),
+				}
+			};
+
+			let blocks = if is_canon(&filter.from_block) && is_canon(&filter.to_block) {
+				// If we are on the canon chain, use bloom filter to fetch required hashes.
+				let from = self.block_number_ref(&filter.from_block)?;
+				let to = self.block_number_ref(&filter.to_block)?;
+
+				filter.bloom_possibilities().iter()
+					.map(|bloom| {
+						chain.blocks_with_bloom(bloom, from, to)
+					})
+					.flat_map(|m| m)
+					// remove duplicate elements
+					.collect::<BTreeSet<u64>>()
+					.into_iter()
+					.filter_map(|n| chain.block_hash(n))
+					.collect::<Vec<H256>>()
+
+			} else {
+				// Otherwise, we use a slower version that finds a link between from_block and to_block.
+				let from_hash = Self::block_hash(&chain, filter.from_block)?;
+				let from_number = chain.block_number(&from_hash)?;
+				let to_hash = Self::block_hash(&chain, filter.from_block)?;
+
+				let blooms = filter.bloom_possibilities();
+				let bloom_match = |header: &encoded::Header| {
+					blooms.iter().any(|bloom| header.log_bloom().contains_bloom(bloom))
+				};
+
+				let (blocks, last_hash) = {
+					let mut blocks = Vec::new();
+					let mut current_hash = to_hash;
+
+					loop {
+						let header = chain.block_header_data(&current_hash)?;
+						if bloom_match(&header) {
+							blocks.push(current_hash);
+						}
+
+						// Stop if `from` block is reached.
+						if header.number() <= from_number {
+							break;
+						}
+						current_hash = header.parent_hash();
+					}
+
+					blocks.reverse();
+					(blocks, current_hash)
+				};
+
+				// Check if we've actually reached the expected `from` block.
+				if last_hash != from_hash || blocks.is_empty() {
+					return None;
+				}
+
+				blocks
+			};
+
+			Some(self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit))
 		};
 
-		let chain = self.chain.read();
-		let blocks = filter.bloom_possibilities().iter()
-			.map(move |bloom| {
-				chain.blocks_with_bloom(bloom, from, to)
-			})
-			.flat_map(|m| m)
-			// remove duplicate elements
-			.collect::<HashSet<u64>>()
-			.into_iter()
-			.collect::<Vec<u64>>();
-
-		self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit)
+		fetch_logs().unwrap_or_default()
 	}
 
 	fn filter_traces(&self, filter: TraceFilter) -> Option<Vec<LocalizedTrace>> {
@@ -1917,35 +1894,10 @@ impl BlockChainClient for Client {
 		(*self.build_last_hashes(&self.chain.read().best_block_hash())).clone()
 	}
 
-	fn queue_transactions(&self, transactions: Vec<Bytes>, peer_id: usize) {
-		let queue_size = self.queue_transactions.load(AtomicOrdering::Relaxed);
-		trace!(target: "external_tx", "Queue size: {}", queue_size);
-		if queue_size > MAX_TX_QUEUE_SIZE {
-			debug!("Ignoring {} transactions: queue is full", transactions.len());
-		} else {
-			let len = transactions.len();
-			match self.io_channel.lock().send(ClientIoMessage::NewTransactions(transactions, peer_id)) {
-				Ok(_) => {
-					self.queue_transactions.fetch_add(len, AtomicOrdering::SeqCst);
-				}
-				Err(e) => {
-					debug!("Ignoring {} transactions: error queueing: {}", len, e);
-				}
-			}
-		}
-	}
-
 	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>> {
 		self.importer.miner.ready_transactions(self)
 	}
 
-	fn queue_consensus_message(&self, message: Bytes) {
-		let channel = self.io_channel.lock().clone();
-		if let Err(e) = channel.send(ClientIoMessage::NewMessage(message)) {
-			debug!("Ignoring the message, error queueing: {}", e);
-		}
-	}
-
 	fn signing_chain_id(&self) -> Option<u64> {
 		self.engine.signing_chain_id(&self.latest_env_info())
 	}
@@ -1957,7 +1909,11 @@ impl BlockChainClient for Client {
 
 	fn uncle_extra_info(&self, id: UncleId) -> Option<BTreeMap<String, String>> {
 		self.uncle(id)
-			.map(|header| self.engine.extra_info(&header.decode()))
+			.and_then(|h| {
+				h.decode().map(|dh| {
+					self.engine.extra_info(&dh)
+				}).ok()
+			})
 	}
 
 	fn pruning_info(&self) -> PruningInfo {
@@ -1993,6 +1949,104 @@ impl BlockChainClient for Client {
 	}
 }
 
+impl IoClient for Client {
+	fn queue_transactions(&self, transactions: Vec<Bytes>, peer_id: usize) {
+		trace_time!("queue_transactions");
+		let len = transactions.len();
+		self.queue_transactions.queue(&mut self.io_channel.lock(), len, move |client| {
+			trace_time!("import_queued_transactions");
+
+			let txs: Vec<UnverifiedTransaction> = transactions
+				.iter()
+				.filter_map(|bytes| client.engine.decode_transaction(bytes).ok())
+				.collect();
+
+			client.notify(|notify| {
+				notify.transactions_received(&txs, peer_id);
+			});
+
+			client.importer.miner.import_external_transactions(client, txs);
+		}).unwrap_or_else(|e| {
+			debug!(target: "client", "Ignoring {} transactions: {}", len, e);
+		});
+	}
+
+	fn queue_ancient_block(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, BlockImportError> {
+		trace_time!("queue_ancient_block");
+		let header: Header = ::rlp::Rlp::new(&block_bytes).val_at(0)?;
+		let hash = header.hash();
+
+		{
+			// check block order
+			if self.chain.read().is_known(&hash) {
+				bail!(BlockImportErrorKind::Import(ImportErrorKind::AlreadyInChain));
+			}
+			let parent_hash = header.parent_hash();
+			// NOTE To prevent race condition with import, make sure to check queued blocks first
+			// (and attempt to acquire lock)
+			let is_parent_pending = self.queued_ancient_blocks.read().0.contains(parent_hash);
+			if !is_parent_pending {
+				let status = self.block_status(BlockId::Hash(*parent_hash));
+				if  status == BlockStatus::Unknown || status == BlockStatus::Pending {
+					bail!(BlockImportErrorKind::Block(BlockError::UnknownParent(*parent_hash)));
+				}
+			}
+		}
+
+		// we queue blocks here and trigger an IO message.
+		{
+			let mut queued = self.queued_ancient_blocks.write();
+			queued.0.insert(hash);
+			queued.1.push_back((header, block_bytes, receipts_bytes));
+		}
+
+		let queued = self.queued_ancient_blocks.clone();
+		let lock = self.ancient_blocks_import_lock.clone();
+		match self.queue_ancient_blocks.queue(&mut self.io_channel.lock(), 1, move |client| {
+			trace_time!("import_ancient_block");
+			// Make sure to hold the lock here to prevent importing out of order.
+			// We use separate lock, cause we don't want to block queueing.
+			let _lock = lock.lock();
+			for _i in 0..MAX_ANCIENT_BLOCKS_TO_IMPORT {
+				let first = queued.write().1.pop_front();
+				if let Some((header, block_bytes, receipts_bytes)) = first {
+					let hash = header.hash();
+					let result = client.importer.import_old_block(
+						&header,
+						&block_bytes,
+						&receipts_bytes,
+						&**client.db.read(),
+						&*client.chain.read(),
+					);
+					if let Err(e) = result {
+						error!(target: "client", "Error importing ancient block: {}", e);
+					}
+					// remove from pending
+					queued.write().0.remove(&hash);
+				} else {
+					break;
+				}
+			}
+		}) {
+			Ok(_) => Ok(hash),
+			Err(e) => bail!(BlockImportErrorKind::Other(format!("{}", e))),
+		}
+	}
+
+	fn queue_consensus_message(&self, message: Bytes) {
+		match self.queue_consensus_message.queue(&mut self.io_channel.lock(), 1, move |client| {
+			if let Err(e) = client.engine().handle_message(&message) {
+				debug!(target: "poa", "Invalid message received: {}", e);
+			}
+		}) {
+			Ok(_) => (),
+			Err(e) => {
+				debug!(target: "poa", "Ignoring the message, error queueing: {}", e);
+			}
+		}
+	}
+}
+
 impl ReopenBlock for Client {
 	fn reopen_block(&self, block: ClosedBlock) -> OpenBlock {
 		let engine = &*self.engine;
@@ -2009,7 +2063,8 @@ impl ReopenBlock for Client {
 			for h in uncles {
 				if !block.uncles().iter().any(|header| header.hash() == h) {
 					let uncle = chain.block_header_data(&h).expect("find_uncle_hashes only returns hashes for existing headers; qed");
-					block.push_uncle(uncle.decode()).expect("pushing up to maximum_uncle_count;
+					let uncle = uncle.decode().expect("decoding failure");
+					block.push_uncle(uncle).expect("pushing up to maximum_uncle_count;
 												push_uncle is not ok only if more than maximum_uncle_count is pushed;
 												so all push_uncle are Ok;
 												qed");
@@ -2050,7 +2105,7 @@ impl PrepareOpenBlock for Client {
 			.into_iter()
 			.take(engine.maximum_uncle_count(open_block.header().number()))
 			.foreach(|h| {
-				open_block.push_uncle(h.decode()).expect("pushing maximum_uncle_count;
+				open_block.push_uncle(h.decode().expect("decoding failure")).expect("pushing maximum_uncle_count;
 												open_block was just created;
 												push_uncle is not ok only if more than maximum_uncle_count is pushed;
 												so all push_uncle are Ok;
@@ -2087,20 +2142,16 @@ impl ImportSealedBlock for Client {
 			self.state_db.write().sync_cache(&route.enacted, &route.retracted, false);
 			route
 		};
-		let (enacted, retracted) = self.importer.calculate_enacted_retracted(&[route]);
-		self.importer.miner.chain_new_blocks(self, &[h.clone()], &[], &enacted, &retracted, true);
+		let route = ChainRoute::from([route].as_ref());
+		self.importer.miner.chain_new_blocks(self, &[h.clone()], &[], route.enacted(), route.retracted(), true);
 		self.notify(|notify| {
 			notify.new_blocks(
 				vec![h.clone()],
 				vec![],
-				enacted.clone(),
-				retracted.clone(),
+				route.clone(),
 				vec![h.clone()],
 				vec![],
-				{
-					let elapsed = start.elapsed();
-					elapsed.as_secs() * 1_000_000_000 + elapsed.subsec_nanos() as u64
-				},
+				start.elapsed(),
 			);
 		});
 		self.db.read().flush().expect("DB flush failed.");
@@ -2110,15 +2161,15 @@ impl ImportSealedBlock for Client {
 
 impl BroadcastProposalBlock for Client {
 	fn broadcast_proposal_block(&self, block: SealedBlock) {
+		const DURATION_ZERO: Duration = Duration::from_millis(0);
 		self.notify(|notify| {
 			notify.new_blocks(
 				vec![],
 				vec![],
-				vec![],
-				vec![],
+				ChainRoute::default(),
 				vec![],
 				vec![block.rlp_bytes()],
-				0,
+				DURATION_ZERO,
 			);
 		});
 	}
@@ -2372,3 +2423,54 @@ mod tests {
 		});
 	}
 }
+
+#[derive(Debug)]
+enum QueueError {
+	Channel(IoError),
+	Full(usize),
+}
+
+impl fmt::Display for QueueError {
+	fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			QueueError::Channel(ref c) => fmt::Display::fmt(c, fmt),
+			QueueError::Full(limit) => write!(fmt, "The queue is full ({})", limit),
+		}
+	}
+}
+
+/// Queue some items to be processed by IO client.
+struct IoChannelQueue {
+	currently_queued: Arc<AtomicUsize>,
+	limit: usize,
+}
+
+impl IoChannelQueue {
+	pub fn new(limit: usize) -> Self {
+		IoChannelQueue {
+			currently_queued: Default::default(),
+			limit,
+		}
+	}
+
+	pub fn queue<F>(&self, channel: &mut IoChannel<ClientIoMessage>, count: usize, fun: F) -> Result<(), QueueError> where
+		F: Fn(&Client) + Send + Sync + 'static,
+	{
+		let queue_size = self.currently_queued.load(AtomicOrdering::Relaxed);
+		ensure!(queue_size < self.limit, QueueError::Full(self.limit));
+
+		let currently_queued = self.currently_queued.clone();
+		let result = channel.send(ClientIoMessage::execute(move |client| {
+			currently_queued.fetch_sub(count, AtomicOrdering::SeqCst);
+			fun(client);
+		}));
+
+		match result {
+			Ok(_) => {
+				self.currently_queued.fetch_add(count, AtomicOrdering::SeqCst);
+				Ok(())
+			},
+			Err(e) => Err(QueueError::Channel(e)),
+		}
+	}
+}

ethcore/src/client/io_message.rs

@@ -14,19 +14,19 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use ethereum_types::H256;
+use std::fmt;
 use bytes::Bytes;
+use client::Client;
+use ethereum_types::H256;
 use snapshot::ManifestData;
 
 /// Message type for external and internal events
-#[derive(Clone, PartialEq, Eq, Debug)]
+#[derive(Debug)]
 pub enum ClientIoMessage {
 	/// Best Block Hash in chain has been changed
 	NewChainHead,
 	/// A block is ready
 	BlockVerified,
-	/// New transaction RLPs are ready to be imported
-	NewTransactions(Vec<Bytes>, usize),
 	/// Begin snapshot restoration
 	BeginRestoration(ManifestData),
 	/// Feed a state chunk to the snapshot service
@@ -35,9 +35,23 @@ pub enum ClientIoMessage {
 	FeedBlockChunk(H256, Bytes),
 	/// Take a snapshot for the block with given number.
 	TakeSnapshot(u64),
-	/// New consensus message received.
-	NewMessage(Bytes),
-	/// New private transaction arrived
-	NewPrivateTransaction,
+	/// Execute wrapped closure
+	Execute(Callback),
+}
+
+impl ClientIoMessage {
+	/// Create new `ClientIoMessage` that executes given procedure.
+	pub fn execute<F: Fn(&Client) + Send + Sync + 'static>(fun: F) -> Self {
+		ClientIoMessage::Execute(Callback(Box::new(fun)))
+	}
+}
+
+/// A function to invoke in the client thread.
+pub struct Callback(pub Box<Fn(&Client) + Send + Sync>);
+
+impl fmt::Debug for Callback {
+	fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+		write!(fmt, "<callback>")
+	}
 }
 

ethcore/src/client/mod.rs

@@ -31,14 +31,13 @@ pub use self::error::Error;
 pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactResult};
 pub use self::io_message::ClientIoMessage;
 pub use self::test_client::{TestBlockChainClient, EachBlockWith};
-pub use self::chain_notify::{ChainNotify, ChainMessageType};
+pub use self::chain_notify::{ChainNotify, ChainRoute, ChainRouteType, ChainMessageType};
 pub use self::traits::{
     Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, PrepareOpenBlock, CallContract, TransactionInfo, RegistryInfo, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock,
     StateOrBlock, StateClient, Call, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter
 };
-//pub use self::private_notify::PrivateNotify;
 pub use state::StateInfo;
-pub use self::traits::{BlockChainClient, EngineClient, ProvingBlockChainClient};
+pub use self::traits::{BlockChainClient, EngineClient, ProvingBlockChainClient, IoClient};
 
 pub use types::ids::*;
 pub use types::trace_filter::Filter as TraceFilter;

ethcore/src/client/test_client.rs

@@ -39,7 +39,7 @@ use client::{
 	PrepareOpenBlock, BlockChainClient, BlockChainInfo, BlockStatus, BlockId,
 	TransactionId, UncleId, TraceId, TraceFilter, LastHashes, CallAnalytics, BlockImportError,
 	ProvingBlockChainClient, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock,
-	Call, StateClient, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter
+	Call, StateClient, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter, IoClient
 };
 use db::{NUM_COLUMNS, COL_STATE};
 use header::{Header as BlockHeader, BlockNumber};
@@ -289,7 +289,7 @@ impl TestBlockChainClient {
 	/// Make a bad block by setting invalid extra data.
 	pub fn corrupt_block(&self, n: BlockNumber) {
 		let hash = self.block_hash(BlockId::Number(n)).unwrap();
-		let mut header: BlockHeader = self.block_header(BlockId::Number(n)).unwrap().decode();
+		let mut header: BlockHeader = self.block_header(BlockId::Number(n)).unwrap().decode().expect("decoding failed");
 		header.set_extra_data(b"This extra data is way too long to be considered valid".to_vec());
 		let mut rlp = RlpStream::new_list(3);
 		rlp.append(&header);
@@ -301,7 +301,7 @@ impl TestBlockChainClient {
 	/// Make a bad block by setting invalid parent hash.
 	pub fn corrupt_block_parent(&self, n: BlockNumber) {
 		let hash = self.block_hash(BlockId::Number(n)).unwrap();
-		let mut header: BlockHeader = self.block_header(BlockId::Number(n)).unwrap().decode();
+		let mut header: BlockHeader = self.block_header(BlockId::Number(n)).unwrap().decode().expect("decoding failed");
 		header.set_parent_hash(H256::from(42));
 		let mut rlp = RlpStream::new_list(3);
 		rlp.append(&header);
@@ -479,6 +479,7 @@ impl BlockInfo for TestBlockChainClient {
 		self.block_header(BlockId::Hash(self.chain_info().best_block_hash))
 			.expect("Best block always has header.")
 			.decode()
+			.expect("decoding failed")
 	}
 
 	fn block(&self, id: BlockId) -> Option<encoded::Block> {
@@ -556,10 +557,6 @@ impl ImportBlock for TestBlockChainClient {
 		}
 		Ok(h)
 	}
-
-	fn import_block_with_receipts(&self, b: Bytes, _r: Bytes) -> Result<H256, BlockImportError> {
-		self.import_block(b)
-	}
 }
 
 impl Call for TestBlockChainClient {
@@ -809,16 +806,6 @@ impl BlockChainClient for TestBlockChainClient {
 		self.traces.read().clone()
 	}
 
-	fn queue_transactions(&self, transactions: Vec<Bytes>, _peer_id: usize) {
-		// import right here
-		let txs = transactions.into_iter().filter_map(|bytes| Rlp::new(&bytes).as_val().ok()).collect();
-		self.miner.import_external_transactions(self, txs);
-	}
-
-	fn queue_consensus_message(&self, message: Bytes) {
-		self.spec.engine.handle_message(&message).unwrap();
-	}
-
 	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>> {
 		self.miner.ready_transactions(self)
 	}
@@ -863,6 +850,22 @@ impl BlockChainClient for TestBlockChainClient {
 	fn eip86_transition(&self) -> u64 { u64::max_value() }
 }
 
+impl IoClient for TestBlockChainClient {
+	fn queue_transactions(&self, transactions: Vec<Bytes>, _peer_id: usize) {
+		// import right here
+		let txs = transactions.into_iter().filter_map(|bytes| Rlp::new(&bytes).as_val().ok()).collect();
+		self.miner.import_external_transactions(self, txs);
+	}
+
+	fn queue_ancient_block(&self, b: Bytes, _r: Bytes) -> Result<H256, BlockImportError> {
+		self.import_block(b)
+	}
+
+	fn queue_consensus_message(&self, message: Bytes) {
+		self.spec.engine.handle_message(&message).unwrap();
+	}
+}
+
 impl ProvingBlockChainClient for TestBlockChainClient {
 	fn prove_storage(&self, _: H256, _: H256, _: BlockId) -> Option<(Vec<Bytes>, H256)> {
 		None

ethcore/src/client/traits.rs

@@ -168,9 +168,6 @@ pub trait RegistryInfo {
 pub trait ImportBlock {
 	/// Import a block into the blockchain.
 	fn import_block(&self, bytes: Bytes) -> Result<H256, BlockImportError>;
-
-	/// Import a block with transaction receipts. Does no sealing and transaction validation.
-	fn import_block_with_receipts(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, BlockImportError>;
 }
 
 /// Provides `call_contract` method
@@ -201,8 +198,21 @@ pub trait EngineInfo {
 	fn engine(&self) -> &EthEngine;
 }
 
+/// IO operations that should off-load heavy work to another thread.
+pub trait IoClient: Sync + Send {
+	/// Queue transactions for importing.
+	fn queue_transactions(&self, transactions: Vec<Bytes>, peer_id: usize);
+
+	/// Queue block import with transaction receipts. Does no sealing and transaction validation.
+	fn queue_ancient_block(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, BlockImportError>;
+
+	/// Queue conensus engine message.
+	fn queue_consensus_message(&self, message: Bytes);
+}
+
 /// Blockchain database client. Owns and manages a blockchain and a block queue.
-pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContract + RegistryInfo + ImportBlock {
+pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContract + RegistryInfo + ImportBlock
++ IoClient {
 	/// Look up the block number for the given block ID.
 	fn block_number(&self, id: BlockId) -> Option<BlockNumber>;
 
@@ -310,12 +320,6 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 	/// Get last hashes starting from best block.
 	fn last_hashes(&self) -> LastHashes;
 
-	/// Queue transactions for importing.
-	fn queue_transactions(&self, transactions: Vec<Bytes>, peer_id: usize);
-
-	/// Queue conensus engine message.
-	fn queue_consensus_message(&self, message: Bytes);
-
 	/// List all transactions that are allowed into the next block.
 	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>>;
 

ethcore/src/db.rs

@@ -218,15 +218,12 @@ impl Writable for DBTransaction {
 }
 
 impl<KVDB: KeyValueDB + ?Sized> Readable for KVDB {
-	fn read<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> Option<T> where T: rlp::Decodable, R: Deref<Target = [u8]> {
-		let result = self.get(col, &key.key());
+	fn read<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> Option<T>
+		where T: rlp::Decodable, R: Deref<Target = [u8]> {
+		self.get(col, &key.key())
+			.expect(&format!("db get failed, key: {:?}", &key.key() as &[u8]))
+			.map(|v| rlp::decode(&v).expect("decode db value failed") )
 
-		match result {
-			Ok(option) => option.map(|v| rlp::decode(&v)),
-			Err(err) => {
-				panic!("db get failed, key: {:?}, err: {:?}", &key.key() as &[u8], err);
-			}
-		}
 	}
 
 	fn exists<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> bool where R: Deref<Target = [u8]> {

ethcore/src/encoded.rs

@@ -24,13 +24,12 @@
 //! decoded object where parts like the hash can be saved.
 
 use block::Block as FullBlock;
-use header::{BlockNumber, Header as FullHeader};
-use transaction::UnverifiedTransaction;
-
-use hash::keccak;
-use heapsize::HeapSizeOf;
 use ethereum_types::{H256, Bloom, U256, Address};
-use rlp::{Rlp, RlpStream};
+use hash::keccak;
+use header::{BlockNumber, Header as FullHeader};
+use heapsize::HeapSizeOf;
+use rlp::{self, Rlp, RlpStream};
+use transaction::UnverifiedTransaction;
 use views::{self, BlockView, HeaderView, BodyView};
 
 /// Owning header view.
@@ -48,7 +47,9 @@ impl Header {
 	pub fn new(encoded: Vec<u8>) -> Self { Header(encoded) }
 
 	/// Upgrade this encoded view to a fully owned `Header` object.
-	pub fn decode(&self) -> FullHeader { ::rlp::decode(&self.0) }
+	pub fn decode(&self) -> Result<FullHeader, rlp::DecoderError> {
+		rlp::decode(&self.0)
+	}
 
 	/// Get a borrowed header view onto the data.
 	#[inline]
@@ -205,7 +206,7 @@ impl Block {
 	pub fn header_view(&self) -> HeaderView { self.view().header_view() }
 
 	/// Decode to a full block.
-	pub fn decode(&self) -> FullBlock { ::rlp::decode(&self.0) }
+	pub fn decode(&self) -> Result<FullBlock, rlp::DecoderError> { rlp::decode(&self.0) }
 
 	/// Decode the header.
 	pub fn decode_header(&self) -> FullHeader { self.view().rlp().val_at(0) }

ethcore/src/engines/authority_round/mod.rs

@@ -382,12 +382,16 @@ impl Decodable for SealedEmptyStep {
 	}
 }
 
+struct PermissionedStep {
+	inner: Step,
+	can_propose: AtomicBool,
+}
+
 /// Engine using `AuthorityRound` proof-of-authority BFT consensus.
 pub struct AuthorityRound {
 	transition_service: IoService<()>,
-	step: Arc<Step>,
-	can_propose: AtomicBool,
-	client: RwLock<Option<Weak<EngineClient>>>,
+	step: Arc<PermissionedStep>,
+	client: Arc<RwLock<Option<Weak<EngineClient>>>>,
 	signer: RwLock<EngineSigner>,
 	validators: Box<ValidatorSet>,
 	validate_score_transition: u64,
@@ -407,7 +411,7 @@ pub struct AuthorityRound {
 
 // header-chain validator.
 struct EpochVerifier {
-	step: Arc<Step>,
+	step: Arc<PermissionedStep>,
 	subchain_validators: SimpleList,
 	empty_steps_transition: u64,
 }
@@ -415,7 +419,7 @@ struct EpochVerifier {
 impl super::EpochVerifier<EthereumMachine> for EpochVerifier {
 	fn verify_light(&self, header: &Header) -> Result<(), Error> {
 		// Validate the timestamp
-		verify_timestamp(&*self.step, header_step(header, self.empty_steps_transition)?)?;
+		verify_timestamp(&self.step.inner, header_step(header, self.empty_steps_transition)?)?;
 		// always check the seal since it's fast.
 		// nothing heavier to do.
 		verify_external(header, &self.subchain_validators, self.empty_steps_transition)
@@ -615,13 +619,15 @@ impl AuthorityRound {
 		let engine = Arc::new(
 			AuthorityRound {
 				transition_service: IoService::<()>::start()?,
-				step: Arc::new(Step {
-					inner: AtomicUsize::new(initial_step),
-					calibrate: our_params.start_step.is_none(),
-					duration: our_params.step_duration,
+				step: Arc::new(PermissionedStep {
+					inner: Step {
+						inner: AtomicUsize::new(initial_step),
+						calibrate: our_params.start_step.is_none(),
+						duration: our_params.step_duration,
+					},
+					can_propose: AtomicBool::new(true),
 				}),
-				can_propose: AtomicBool::new(true),
-				client: RwLock::new(None),
+				client: Arc::new(RwLock::new(None)),
 				signer: Default::default(),
 				validators: our_params.validators,
 				validate_score_transition: our_params.validate_score_transition,
@@ -641,7 +647,10 @@ impl AuthorityRound {
 
 		// Do not initialize timeouts for tests.
 		if should_timeout {
-			let handler = TransitionHandler { engine: Arc::downgrade(&engine) };
+			let handler = TransitionHandler {
+				step: engine.step.clone(),
+				client: engine.client.clone(),
+			};
 			engine.transition_service.register_handler(Arc::new(handler))?;
 		}
 		Ok(engine)
@@ -667,7 +676,7 @@ impl AuthorityRound {
 	}
 
 	fn generate_empty_step(&self, parent_hash: &H256) {
-		let step = self.step.load();
+		let step = self.step.inner.load();
 		let empty_step_rlp = empty_step_rlp(step, parent_hash);
 
 		if let Ok(signature) = self.sign(keccak(&empty_step_rlp)).map(Into::into) {
@@ -699,34 +708,37 @@ fn unix_now() -> Duration {
 }
 
 struct TransitionHandler {
-	engine: Weak<AuthorityRound>,
+	step: Arc<PermissionedStep>,
+	client: Arc<RwLock<Option<Weak<EngineClient>>>>,
 }
 
 const ENGINE_TIMEOUT_TOKEN: TimerToken = 23;
 
 impl IoHandler<()> for TransitionHandler {
 	fn initialize(&self, io: &IoContext<()>) {
-		if let Some(engine) = self.engine.upgrade() {
-			let remaining = engine.step.duration_remaining().as_millis();
-			io.register_timer_once(ENGINE_TIMEOUT_TOKEN, Duration::from_millis(remaining))
-				.unwrap_or_else(|e| warn!(target: "engine", "Failed to start consensus step timer: {}.", e))
-		}
+		let remaining = self.step.inner.duration_remaining().as_millis();
+		io.register_timer_once(ENGINE_TIMEOUT_TOKEN, Duration::from_millis(remaining))
+			.unwrap_or_else(|e| warn!(target: "engine", "Failed to start consensus step timer: {}.", e))
 	}
 
 	fn timeout(&self, io: &IoContext<()>, timer: TimerToken) {
 		if timer == ENGINE_TIMEOUT_TOKEN {
-			if let Some(engine) = self.engine.upgrade() {
-				// NOTE we might be lagging by couple of steps in case the timeout
-				// has not been called fast enough.
-				// Make sure to advance up to the actual step.
-				while engine.step.duration_remaining().as_millis() == 0 {
-					engine.step();
+			// NOTE we might be lagging by couple of steps in case the timeout
+			// has not been called fast enough.
+			// Make sure to advance up to the actual step.
+			while self.step.inner.duration_remaining().as_millis() == 0 {
+				self.step.inner.increment();
+				self.step.can_propose.store(true, AtomicOrdering::SeqCst);
+				if let Some(ref weak) = *self.client.read() {
+					if let Some(c) = weak.upgrade() {
+						c.update_sealing();
+					}
 				}
-
-				let next_run_at = engine.step.duration_remaining().as_millis() >> 2;
-				io.register_timer_once(ENGINE_TIMEOUT_TOKEN, Duration::from_millis(next_run_at))
-					.unwrap_or_else(|e| warn!(target: "engine", "Failed to restart consensus step timer: {}.", e))
 			}
+
+			let next_run_at = self.step.inner.duration_remaining().as_millis() >> 2;
+			io.register_timer_once(ENGINE_TIMEOUT_TOKEN, Duration::from_millis(next_run_at))
+				.unwrap_or_else(|e| warn!(target: "engine", "Failed to restart consensus step timer: {}.", e))
 		}
 	}
 }
@@ -743,8 +755,8 @@ impl Engine<EthereumMachine> for AuthorityRound {
 	}
 
 	fn step(&self) {
-		self.step.increment();
-		self.can_propose.store(true, AtomicOrdering::SeqCst);
+		self.step.inner.increment();
+		self.step.can_propose.store(true, AtomicOrdering::SeqCst);
 		if let Some(ref weak) = *self.client.read() {
 			if let Some(c) = weak.upgrade() {
 				c.update_sealing();
@@ -791,7 +803,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 
 	fn populate_from_parent(&self, header: &mut Header, parent: &Header) {
 		let parent_step = header_step(parent, self.empty_steps_transition).expect("Header has been verified; qed");
-		let current_step = self.step.load();
+		let current_step = self.step.inner.load();
 
 		let current_empty_steps_len = if header.number() >= self.empty_steps_transition {
 			self.empty_steps(parent_step.into(), current_step.into(), parent.hash()).len()
@@ -817,7 +829,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		let empty_step: EmptyStep = rlp.as_val().map_err(fmt_err)?;;
 
 		if empty_step.verify(&*self.validators).unwrap_or(false) {
-			if self.step.check_future(empty_step.step).is_ok() {
+			if self.step.inner.check_future(empty_step.step).is_ok() {
 				trace!(target: "engine", "handle_message: received empty step message {:?}", empty_step);
 				self.handle_empty_step_message(empty_step);
 			} else {
@@ -837,7 +849,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 	fn generate_seal(&self, block: &ExecutedBlock, parent: &Header) -> Seal {
 		// first check to avoid generating signature most of the time
 		// (but there's still a race to the `compare_and_swap`)
-		if !self.can_propose.load(AtomicOrdering::SeqCst) {
+		if !self.step.can_propose.load(AtomicOrdering::SeqCst) {
 			trace!(target: "engine", "Aborting seal generation. Can't propose.");
 			return Seal::None;
 		}
@@ -846,7 +858,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		let parent_step: U256 = header_step(parent, self.empty_steps_transition)
 			.expect("Header has been verified; qed").into();
 
-		let step = self.step.load();
+		let step = self.step.inner.load();
 
 		// filter messages from old and future steps and different parents
 		let empty_steps = if header.number() >= self.empty_steps_transition {
@@ -923,7 +935,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 				trace!(target: "engine", "generate_seal: Issuing a block for step {}.", step);
 
 				// only issue the seal if we were the first to reach the compare_and_swap.
-				if self.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) {
+				if self.step.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) {
 
 					self.clear_empty_steps(parent_step);
 
@@ -996,10 +1008,10 @@ impl Engine<EthereumMachine> for AuthorityRound {
 
 				let parent = client.block_header(::client::BlockId::Hash(*block.header().parent_hash()))
 					.expect("hash is from parent; parent header must exist; qed")
-					.decode();
+					.decode()?;
 
 				let parent_step = header_step(&parent, self.empty_steps_transition)?;
-				let current_step = self.step.load();
+				let current_step = self.step.inner.load();
 				self.empty_steps(parent_step.into(), current_step.into(), parent.hash())
 			} else {
 				// we're verifying a block, extract empty steps from the seal
@@ -1049,7 +1061,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 		// If yes then probably benign reporting needs to be moved further in the verification.
 		let set_number = header.number();
 
-		match verify_timestamp(&*self.step, header_step(header, self.empty_steps_transition)?) {
+		match verify_timestamp(&self.step.inner, header_step(header, self.empty_steps_transition)?) {
 			Err(BlockError::InvalidSeal) => {
 				self.validators.report_benign(header.author(), set_number, header.number());
 				Err(BlockError::InvalidSeal.into())
@@ -1291,7 +1303,7 @@ impl Engine<EthereumMachine> for AuthorityRound {
 						// This way, upon encountering an epoch change, the proposer from the
 						// new set will be forced to wait until the next step to avoid sealing a
 						// block that breaks the invariant that the parent's step < the block's step.
-						self.can_propose.store(false, AtomicOrdering::SeqCst);
+						self.step.can_propose.store(false, AtomicOrdering::SeqCst);
 						return Some(combine_proofs(signal_number, &pending.proof, &*finality_proof));
 					}
 				}

ethcore/src/engines/mod.rs

@@ -426,6 +426,11 @@ pub trait EthEngine: Engine<::machine::EthereumMachine> {
 	fn additional_params(&self) -> HashMap<String, String> {
 		self.machine().additional_params()
 	}
+
+	/// Performs pre-validation of RLP decoded transaction before other processing
+	fn decode_transaction(&self, transaction: &[u8]) -> Result<UnverifiedTransaction, transaction::Error> {
+		self.machine().decode_transaction(transaction)
+	}
 }
 
 // convenience wrappers for existing functions.

ethcore/src/engines/tendermint/mod.rs

@@ -142,8 +142,10 @@ impl <F> super::EpochVerifier<EthereumMachine> for EpochVerifier<F>
 	}
 
 	fn check_finality_proof(&self, proof: &[u8]) -> Option<Vec<H256>> {
-		let header: Header = ::rlp::decode(proof);
-		self.verify_light(&header).ok().map(|_| vec![header.hash()])
+		match ::rlp::decode(proof) {
+			Ok(header) => self.verify_light(&header).ok().map(|_| vec![header.hash()]),
+			Err(_) => None // REVIEW: log perhaps? Not sure what the policy is.
+		}
 	}
 }
 

ethcore/src/error.rs

@@ -190,6 +190,7 @@ error_chain! {
 
 	foreign_links {
 		Block(BlockError) #[doc = "Block error"];
+		Decoder(::rlp::DecoderError) #[doc = "Rlp decoding error"];
 	}
 
 	errors {
@@ -206,6 +207,7 @@ impl From<Error> for BlockImportError {
 		match e {
 			Error(ErrorKind::Block(block_error), _) => BlockImportErrorKind::Block(block_error).into(),
 			Error(ErrorKind::Import(import_error), _) => BlockImportErrorKind::Import(import_error.into()).into(),
+			Error(ErrorKind::Util(util_error::ErrorKind::Decoder(decoder_err)), _) => BlockImportErrorKind::Decoder(decoder_err).into(),
 			_ => BlockImportErrorKind::Other(format!("other block import error: {:?}", e)).into(),
 		}
 	}
@@ -288,6 +290,12 @@ error_chain! {
 			description("Unknown engine name")
 			display("Unknown engine name ({})", name)
 		}
+
+		#[doc = "RLP decoding errors"]
+		Decoder(err: ::rlp::DecoderError) {
+			description("decoding value failed")
+			display("decoding value failed with error: {}", err)
+		}
 	}
 }
 
@@ -308,11 +316,11 @@ impl From<AccountsError> for Error {
 	fn from(err: AccountsError) -> Error { 
 		ErrorKind::AccountProvider(err).into()
 	} 
-} 
+}
 
 impl From<::rlp::DecoderError> for Error {
 	fn from(err: ::rlp::DecoderError) -> Error {
-		UtilError::from(err).into()
+		ErrorKind::Decoder(err).into()
 	}
 }
 

ethcore/src/ethereum/mod.rs

@@ -101,6 +101,9 @@ pub fn new_morden<'a, T: Into<SpecParams<'a>>>(params: T) -> Spec {
 /// Create a new Foundation Frontier-era chain spec as though it never changes to Homestead.
 pub fn new_frontier_test() -> Spec { load(None, include_bytes!("../../res/ethereum/frontier_test.json")) }
 
+/// Create a new Ropsten chain spec.
+pub fn new_ropsten_test() -> Spec { load(None, include_bytes!("../../res/ethereum/ropsten.json")) }
+
 /// Create a new Foundation Homestead-era chain spec as though it never changed from Frontier.
 pub fn new_homestead_test() -> Spec { load(None, include_bytes!("../../res/ethereum/homestead_test.json")) }
 

ethcore/src/executive.rs

@@ -428,8 +428,14 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> {
 					self.state.discard_checkpoint();
 					output.write(0, &builtin_out_buffer);
 
-					// trace only top level calls to builtins to avoid DDoS attacks
-					if self.depth == 0 {
+					// Trace only top level calls and calls with balance transfer to builtins. The reason why we don't
+					// trace all internal calls to builtin contracts is that memcpy (IDENTITY) is a heavily used
+					// function.
+					let is_transferred = match params.value {
+						ActionValue::Transfer(value) => value != U256::zero(),
+						ActionValue::Apparent(_) => false,
+					};
+					if self.depth == 0 || is_transferred {
 						let mut trace_output = tracer.prepare_trace_output();
 						if let Some(out) = trace_output.as_mut() {
 							*out = output.to_owned();
@@ -722,6 +728,12 @@ mod tests {
 		machine
 	}
 
+	fn make_byzantium_machine(max_depth: usize) -> EthereumMachine {
+		let mut machine = ::ethereum::new_byzantium_test_machine();
+		machine.set_schedule_creation_rules(Box::new(move |s, _| s.max_depth = max_depth));
+		machine
+	}
+
 	#[test]
 	fn test_contract_address() {
 		let address = Address::from_str("0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6").unwrap();
@@ -813,6 +825,76 @@ mod tests {
 		assert_eq!(substate.contracts_created.len(), 0);
 	}
 
+	#[test]
+	fn test_call_to_precompiled_tracing() {
+		// code:
+		//
+		// 60 00 - push 00 out size
+		// 60 00 - push 00 out offset
+		// 60 00 - push 00 in size
+		// 60 00 - push 00 in offset
+		// 60 01 - push 01 value
+		// 60 03 - push 03 to
+		// 61 ffff - push fff gas
+		// f1 - CALL
+
+		let code = "60006000600060006001600361fffff1".from_hex().unwrap();
+		let sender = Address::from_str("4444444444444444444444444444444444444444").unwrap();
+		let address = Address::from_str("5555555555555555555555555555555555555555").unwrap();
+
+		let mut params = ActionParams::default();
+		params.address = address.clone();
+		params.code_address = address.clone();
+		params.sender = sender.clone();
+		params.origin = sender.clone();
+		params.gas = U256::from(100_000);
+		params.code = Some(Arc::new(code));
+		params.value = ActionValue::Transfer(U256::from(100));
+		params.call_type = CallType::Call;
+		let mut state = get_temp_state();
+		state.add_balance(&sender, &U256::from(100), CleanupMode::NoEmpty).unwrap();
+		let info = EnvInfo::default();
+		let machine = make_byzantium_machine(5);
+		let mut substate = Substate::new();
+		let mut tracer = ExecutiveTracer::default();
+		let mut vm_tracer = ExecutiveVMTracer::toplevel();
+
+		let mut ex = Executive::new(&mut state, &info, &machine);
+		let output = BytesRef::Fixed(&mut[0u8;0]);
+		ex.call(params, &mut substate, output, &mut tracer, &mut vm_tracer).unwrap();
+
+		assert_eq!(tracer.drain(), vec![FlatTrace {
+			action: trace::Action::Call(trace::Call {
+				from: "4444444444444444444444444444444444444444".into(),
+				to: "5555555555555555555555555555555555555555".into(),
+				value: 100.into(),
+				gas: 100_000.into(),
+				input: vec![],
+				call_type: CallType::Call
+			}),
+			result: trace::Res::Call(trace::CallResult {
+				gas_used: 33021.into(),
+				output: vec![]
+			}),
+			subtraces: 1,
+			trace_address: Default::default()
+		}, FlatTrace {
+			action: trace::Action::Call(trace::Call {
+				from: "5555555555555555555555555555555555555555".into(),
+				to: "0000000000000000000000000000000000000003".into(),
+				value: 1.into(),
+				gas: 66560.into(),
+				input: vec![],
+				call_type: CallType::Call
+			}), result: trace::Res::Call(trace::CallResult {
+				gas_used: 600.into(),
+				output: vec![]
+			}),
+			subtraces: 0,
+			trace_address: vec![0].into_iter().collect(),
+		}]);
+	}
+
 	#[test]
 	// Tracing is not suported in JIT
 	fn test_call_to_create() {

ethcore/src/header.rs

@@ -398,7 +398,7 @@ mod tests {
 		let nonce = "88ab4e252a7e8c2a23".from_hex().unwrap();
 		let nonce_decoded = "ab4e252a7e8c2a23".from_hex().unwrap();
 
-		let header: Header = rlp::decode(&header_rlp);
+		let header: Header = rlp::decode(&header_rlp).expect("error decoding header");
 		let seal_fields = header.seal.clone();
 		assert_eq!(seal_fields.len(), 2);
 		assert_eq!(seal_fields[0], mix_hash);
@@ -415,7 +415,7 @@ mod tests {
 		// that's rlp of block header created with ethash engine.
 		let header_rlp = "f901f9a0d405da4e66f1445d455195229624e133f5baafe72b5cf7b3c36c12c8146e98b7a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a05fb2b4bfdef7b314451cb138a534d225c922fc0e5fbe25e451142732c3e25c25a088d2ec6b9860aae1a2c3b299f72b6a5d70d7f7ba4722c78f2c49ba96273c2158a007c6fdfa8eea7e86b81f5b0fc0f78f90cc19f4aa60d323151e0cac660199e9a1b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefba82524d84568e932a80a0a0349d8c3df71f1a48a9df7d03fd5f14aeee7d91332c009ecaff0a71ead405bd88ab4e252a7e8c2a23".from_hex().unwrap();
 
-		let header: Header = rlp::decode(&header_rlp);
+		let header: Header = rlp::decode(&header_rlp).expect("error decoding header");
 		let encoded_header = rlp::encode(&header).into_vec();
 
 		assert_eq!(header_rlp, encoded_header);

ethcore/src/machine.rs

@@ -34,6 +34,7 @@ use tx_filter::TransactionFilter;
 
 use ethereum_types::{U256, Address};
 use bytes::BytesRef;
+use rlp::Rlp;
 use vm::{CallType, ActionParams, ActionValue, ParamsType};
 use vm::{EnvInfo, Schedule, CreateContractAddress};
 
@@ -121,7 +122,13 @@ impl EthereumMachine {
 }
 
 impl EthereumMachine {
-	/// Execute a call as the system address.
+	/// Execute a call as the system address. Block environment information passed to the
+	/// VM is modified to have its gas limit bounded at the upper limit of possible used
+	/// gases including this system call, capped at the maximum value able to be
+	/// represented by U256. This system call modifies the block state, but discards other
+	/// information. If suicides, logs or refunds happen within the system call, they
+	/// will not be executed or recorded. Gas used by this system call will not be counted
+	/// on the block.
 	pub fn execute_as_system(
 		&self,
 		block: &mut ExecutedBlock,
@@ -131,7 +138,7 @@ impl EthereumMachine {
 	) -> Result<Vec<u8>, Error> {
 		let env_info = {
 			let mut env_info = block.env_info();
-			env_info.gas_limit = env_info.gas_used + gas;
+			env_info.gas_limit = env_info.gas_used.saturating_add(gas);
 			env_info
 		};
 
@@ -376,6 +383,16 @@ impl EthereumMachine {
 			"registrar".to_owned() => format!("{:x}", self.params.registrar)
 		]
 	}
+
+	/// Performs pre-validation of RLP decoded transaction before other processing
+	pub fn decode_transaction(&self, transaction: &[u8]) -> Result<UnverifiedTransaction, transaction::Error> {
+		let rlp = Rlp::new(&transaction);
+		if rlp.as_raw().len() > self.params().max_transaction_size {
+			debug!("Rejected oversized transaction of {} bytes", rlp.as_raw().len());
+			return Err(transaction::Error::TooBig)
+		}
+		rlp.as_val().map_err(|e| transaction::Error::InvalidRlp(e.to_string()))
+	}
 }
 
 /// Auxiliary data fetcher for an Ethereum machine. In Ethereum-like machines
@@ -486,6 +503,25 @@ mod tests {
 		}
 	}
 
+	#[test]
+	fn should_disallow_unsigned_transactions() {
+		let rlp = "ea80843b9aca0083015f90948921ebb5f79e9e3920abe571004d0b1d5119c154865af3107a400080038080".into();
+		let transaction: UnverifiedTransaction = ::rlp::decode(&::rustc_hex::FromHex::from_hex(rlp).unwrap()).unwrap();
+		let spec = ::ethereum::new_ropsten_test();
+		let ethparams = get_default_ethash_extensions();
+
+		let machine = EthereumMachine::with_ethash_extensions(
+			spec.params().clone(),
+			Default::default(),
+			ethparams,
+		);
+		let mut header = ::header::Header::new();
+		header.set_number(15);
+
+		let res = machine.verify_transaction_basic(&transaction, &header);
+		assert_eq!(res, Err(transaction::Error::InvalidSignature("Crypto error (Invalid EC signature)".into())));
+	}
+
 	#[test]
 	fn ethash_gas_limit_is_multiple_of_determinant() {
 		use ethereum_types::U256;

ethcore/src/miner/miner.rs

@@ -126,6 +126,8 @@ pub struct MinerOptions {
 	pub tx_queue_strategy: PrioritizationStrategy,
 	/// Simple senders penalization.
 	pub tx_queue_penalization: Penalization,
+	/// Do we want to mark transactions recieved locally (e.g. RPC) as local if we don't have the sending account?
+	pub tx_queue_no_unfamiliar_locals: bool,
 	/// Do we refuse to accept service transactions even if sender is certified.
 	pub refuse_service_transactions: bool,
 	/// Transaction pool limits.
@@ -149,6 +151,7 @@ impl Default for MinerOptions {
 			infinite_pending_block: false,
 			tx_queue_strategy: PrioritizationStrategy::GasPriceOnly,
 			tx_queue_penalization: Penalization::Disabled,
+			tx_queue_no_unfamiliar_locals: false,
 			refuse_service_transactions: false,
 			pool_limits: pool::Options {
 				max_count: 8_192,
@@ -528,8 +531,8 @@ impl Miner {
 	}
 
 	/// Attempts to perform internal sealing (one that does not require work) and handles the result depending on the type of Seal.
-	fn seal_and_import_block_internally<C>(&self, chain: &C, block: ClosedBlock) -> bool where
-		C: BlockChain + SealedBlockImporter,
+	fn seal_and_import_block_internally<C>(&self, chain: &C, block: ClosedBlock) -> bool
+		where C: BlockChain + SealedBlockImporter,
 	{
 		{
 			let sealing = self.sealing.lock();
@@ -544,7 +547,12 @@ impl Miner {
 		trace!(target: "miner", "seal_block_internally: attempting internal seal.");
 
 		let parent_header = match chain.block_header(BlockId::Hash(*block.header().parent_hash())) {
-			Some(hdr) => hdr.decode(),
+			Some(h) => {
+				match h.decode() {
+					Ok(decoded_hdr) => decoded_hdr,
+					Err(_) => return false
+				}
+			}
 			None => return false,
 		};
 
@@ -683,6 +691,20 @@ impl Miner {
 		// Return if we restarted
 		prepare_new
 	}
+
+	/// Prepare pending block, check whether sealing is needed, and then update sealing.
+	fn prepare_and_update_sealing<C: miner::BlockChainClient>(&self, chain: &C) {
+		use miner::MinerService;
+
+		// Make sure to do it after transaction is imported and lock is dropped.
+		// We need to create pending block and enable sealing.
+		if self.engine.seals_internally().unwrap_or(false) || !self.prepare_pending_block(chain) {
+			// If new block has not been prepared (means we already had one)
+			// or Engine might be able to seal internally,
+			// we need to update sealing.
+			self.update_sealing(chain);
+		}
+	}
 }
 
 const SEALING_TIMEOUT_IN_BLOCKS : u64 = 5;
@@ -749,12 +771,12 @@ impl miner::MinerService for Miner {
 			transactions.into_iter().map(pool::verifier::Transaction::Unverified).collect(),
 		);
 
+		// --------------------------------------------------------------------------
+		// | NOTE Code below requires sealing locks.                                |
+		// | Make sure to release the locks before calling that method.             |
+		// --------------------------------------------------------------------------
 		if !results.is_empty() && self.options.reseal_on_external_tx &&	self.sealing.lock().reseal_allowed() {
-			// --------------------------------------------------------------------------
-			// | NOTE Code below requires sealing locks.                                |
-			// | Make sure to release the locks before calling that method.             |
-			// --------------------------------------------------------------------------
-			self.update_sealing(chain);
+			self.prepare_and_update_sealing(chain);
 		}
 
 		results
@@ -763,8 +785,9 @@ impl miner::MinerService for Miner {
 	fn import_own_transaction<C: miner::BlockChainClient>(
 		&self,
 		chain: &C,
-		pending: PendingTransaction,
+		pending: PendingTransaction
 	) -> Result<(), transaction::Error> {
+		// note: you may want to use `import_claimed_local_transaction` instead of this one.
 
 		trace!(target: "own_tx", "Importing transaction: {:?}", pending);
 
@@ -779,19 +802,34 @@ impl miner::MinerService for Miner {
 		// | Make sure to release the locks before calling that method.             |
 		// --------------------------------------------------------------------------
 		if imported.is_ok() && self.options.reseal_on_own_tx && self.sealing.lock().reseal_allowed() {
-			// Make sure to do it after transaction is imported and lock is droped.
-			// We need to create pending block and enable sealing.
-			if self.engine.seals_internally().unwrap_or(false) || !self.prepare_pending_block(chain) {
-				// If new block has not been prepared (means we already had one)
-				// or Engine might be able to seal internally,
-				// we need to update sealing.
-				self.update_sealing(chain);
-			}
+			self.prepare_and_update_sealing(chain);
 		}
 
 		imported
 	}
 
+	fn import_claimed_local_transaction<C: miner::BlockChainClient>(
+		&self,
+		chain: &C,
+		pending: PendingTransaction,
+		trusted: bool
+	) -> Result<(), transaction::Error> {
+		// treat the tx as local if the option is enabled, or if we have the account
+		let sender = pending.sender();
+		let treat_as_local = trusted
+			|| !self.options.tx_queue_no_unfamiliar_locals
+			|| self.accounts.as_ref().map(|accts| accts.has_account(sender)).unwrap_or(false);
+
+		if treat_as_local {
+			self.import_own_transaction(chain, pending)
+		} else {
+			// We want to replicate behaviour for external transactions if we're not going to treat
+			// this as local. This is important with regards to sealing blocks
+			self.import_external_transactions(chain, vec![pending.transaction.into()])
+				.pop().expect("one result per tx, as in `import_own_transaction`")
+		}
+	}
+
 	fn local_transactions(&self) -> BTreeMap<H256, pool::local_transactions::Status> {
 		self.transaction_queue.local_transactions()
 	}
@@ -1128,6 +1166,7 @@ mod tests {
 				infinite_pending_block: false,
 				tx_queue_penalization: Penalization::Disabled,
 				tx_queue_strategy: PrioritizationStrategy::GasPriceOnly,
+				tx_queue_no_unfamiliar_locals: false,
 				refuse_service_transactions: false,
 				pool_limits: Default::default(),
 				pool_verification_options: pool::verifier::Options {
@@ -1142,8 +1181,10 @@ mod tests {
 		)
 	}
 
+	const TEST_CHAIN_ID: u64 = 2;
+
 	fn transaction() -> SignedTransaction {
-		transaction_with_chain_id(2)
+		transaction_with_chain_id(TEST_CHAIN_ID)
 	}
 
 	fn transaction_with_chain_id(chain_id: u64) -> SignedTransaction {
@@ -1217,6 +1258,53 @@ mod tests {
 		assert_eq!(miner.ready_transactions(&client).len(), 1);
 	}
 
+	#[test]
+	fn should_treat_unfamiliar_locals_selectively() {
+		// given
+		let keypair = Random.generate().unwrap();
+		let client = TestBlockChainClient::default();
+		let account_provider = AccountProvider::transient_provider();
+		account_provider.insert_account(keypair.secret().clone(), "").expect("can add accounts to the provider we just created");
+
+		let miner = Miner::new(
+			MinerOptions {
+				tx_queue_no_unfamiliar_locals: true,
+				..miner().options
+			},
+			GasPricer::new_fixed(0u64.into()),
+			&Spec::new_test(),
+			Some(Arc::new(account_provider)),
+		);
+		let transaction = transaction();
+		let best_block = 0;
+		// when
+		// This transaction should not be marked as local because our account_provider doesn't have the sender
+		let res = miner.import_claimed_local_transaction(&client, PendingTransaction::new(transaction.clone(), None), false);
+
+		// then
+		// Check the same conditions as `should_import_external_transaction` first. Behaviour should be identical.
+		// That is: it's treated as though we added it through `import_external_transactions`
+		assert_eq!(res.unwrap(), ());
+		assert_eq!(miner.pending_transactions(best_block), None);
+		assert_eq!(miner.pending_receipts(best_block), None);
+		assert_eq!(miner.ready_transactions(&client).len(), 0);
+		assert!(miner.prepare_pending_block(&client));
+		assert_eq!(miner.ready_transactions(&client).len(), 1);
+
+		// when - 2nd part: create a local transaction from account_provider.
+		// Borrow the transaction used before & sign with our generated keypair.
+		let local_transaction = transaction.deconstruct().0.as_unsigned().clone().sign(keypair.secret(), Some(TEST_CHAIN_ID));
+		let res2 = miner.import_claimed_local_transaction(&client, PendingTransaction::new(local_transaction, None), false);
+
+		// then - 2nd part: we add on the results from the last pending block.
+		// This is borrowed from `should_make_pending_block_when_importing_own_transaction` and slightly modified.
+		assert_eq!(res2.unwrap(), ());
+		assert_eq!(miner.pending_transactions(best_block).unwrap().len(), 2);
+		assert_eq!(miner.pending_receipts(best_block).unwrap().len(), 2);
+		assert_eq!(miner.ready_transactions(&client).len(), 2);
+		assert!(!miner.prepare_pending_block(&client));
+	}
+
 	#[test]
 	fn should_not_seal_unless_enabled() {
 		let miner = miner();

ethcore/src/miner/mod.rs

@@ -139,6 +139,12 @@ pub trait MinerService : Send + Sync {
 		-> Result<(), transaction::Error>
 		where C: BlockChainClient;
 
+	/// Imports transactions from potentially external sources, with behaviour determined
+	/// by the config flag `tx_queue_allow_unfamiliar_locals`
+	fn import_claimed_local_transaction<C>(&self, chain: &C, transaction: PendingTransaction, trusted: bool)
+		-> Result<(), transaction::Error>
+		where C: BlockChainClient;
+
 	/// Removes transaction from the pool.
 	///
 	/// Attempts to "cancel" a transaction. If it was not propagated yet (or not accepted by other peers)

ethcore/src/miner/pool_client.rs

@@ -124,7 +124,7 @@ impl<'a, C: 'a> pool::client::Client for PoolClient<'a, C> where
 		pool::client::AccountDetails {
 			nonce: self.cached_nonces.account_nonce(address),
 			balance: self.chain.latest_balance(address),
-			is_local: self.accounts.map_or(false, |accounts| accounts.has_account(*address).unwrap_or(false)),
+			is_local: self.accounts.map_or(false, |accounts| accounts.has_account(*address)),
 		}
 	}
 
@@ -145,6 +145,10 @@ impl<'a, C: 'a> pool::client::Client for PoolClient<'a, C> where
 			}
 		}
 	}
+
+	fn decode_transaction(&self, transaction: &[u8]) -> Result<UnverifiedTransaction, transaction::Error> {
+		self.engine.decode_transaction(transaction)
+	}
 }
 
 impl<'a, C: 'a> NonceClient for PoolClient<'a, C> where

ethcore/src/snapshot/account.rs

@@ -236,7 +236,7 @@ mod tests {
 		};
 
 		let thin_rlp = ::rlp::encode(&account);
-		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
+		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
 
 		let fat_rlps = to_fat_rlps(&keccak(&addr), &account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value(), usize::max_value()).unwrap();
 		let fat_rlp = Rlp::new(&fat_rlps[0]).at(1).unwrap();
@@ -261,7 +261,7 @@ mod tests {
 		};
 
 		let thin_rlp = ::rlp::encode(&account);
-		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
+		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
 
 		let fat_rlp = to_fat_rlps(&keccak(&addr), &account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value(), usize::max_value()).unwrap();
 		let fat_rlp = Rlp::new(&fat_rlp[0]).at(1).unwrap();
@@ -286,7 +286,7 @@ mod tests {
 		};
 
 		let thin_rlp = ::rlp::encode(&account);
-		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
+		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp).unwrap(), account);
 
 		let fat_rlps = to_fat_rlps(&keccak(addr), &account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), 500, 1000).unwrap();
 		let mut root = KECCAK_NULL_RLP;

ethcore/src/snapshot/consensus/authority.rs

@@ -100,7 +100,7 @@ impl SnapshotComponents for PoaSnapshot {
 		let (block, receipts) = chain.block(&block_at)
 			.and_then(|b| chain.block_receipts(&block_at).map(|r| (b, r)))
 			.ok_or(Error::BlockNotFound(block_at))?;
-		let block = block.decode();
+		let block = block.decode()?;
 
 		let parent_td = chain.block_details(block.header.parent_hash())
 			.map(|d| d.total_difficulty)

ethcore/src/snapshot/mod.rs

@@ -281,7 +281,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
 	// account_key here is the address' hash.
 	for item in account_trie.iter()? {
 		let (account_key, account_data) = item?;
-		let account = ::rlp::decode(&*account_data);
+		let account = ::rlp::decode(&*account_data)?;
 		let account_key_hash = H256::from_slice(&account_key);
 
 		let account_db = AccountDB::from_hash(db, account_key_hash);
@@ -467,10 +467,10 @@ fn rebuild_accounts(
 		*out = (hash, thin_rlp);
 	}
 	if let Some(&(ref hash, ref rlp)) = out_chunk.iter().last() {
-		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp).storage_root);
+		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp)?.storage_root);
 	}
 	if let Some(&(ref hash, ref rlp)) = out_chunk.iter().next() {
-		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp).storage_root);
+		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp)?.storage_root);
 	}
 	Ok(status)
 }
@@ -487,7 +487,7 @@ pub fn verify_old_block(rng: &mut OsRng, header: &Header, engine: &EthEngine, ch
 	if always || rng.gen::<f32>() <= POW_VERIFY_RATE {
 		engine.verify_block_unordered(header)?;
 		match chain.block_header_data(header.parent_hash()) {
-			Some(parent) => engine.verify_block_family(header, &parent.decode()),
+			Some(parent) => engine.verify_block_family(header, &parent.decode()?),
 			None => Ok(()),
 		}
 	} else {

ethcore/src/snapshot/service.rs

@@ -17,8 +17,8 @@
 //! Snapshot network service implementation.
 
 use std::collections::HashSet;
-use std::io::ErrorKind;
-use std::fs;
+use std::io::{self, Read, ErrorKind};
+use std::fs::{self, File};
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -30,6 +30,7 @@ use blockchain::BlockChain;
 use client::{Client, ChainInfo, ClientIoMessage};
 use engines::EthEngine;
 use error::Error;
+use hash::keccak;
 use ids::BlockId;
 
 use io::IoChannel;
@@ -270,8 +271,8 @@ impl Service {
 			}
 		}
 
-		// delete the temporary restoration dir if it does exist.
-		if let Err(e) = fs::remove_dir_all(service.restoration_dir()) {
+		// delete the temporary restoration DB dir if it does exist.
+		if let Err(e) = fs::remove_dir_all(service.restoration_db()) {
 			if e.kind() != ErrorKind::NotFound {
 				return Err(e.into())
 			}
@@ -325,6 +326,13 @@ impl Service {
 		dir
 	}
 
+	// previous snapshot chunks path.
+	fn prev_chunks_dir(&self) -> PathBuf {
+		let mut dir = self.snapshot_root.clone();
+		dir.push("prev_chunks");
+		dir
+	}
+
 	// replace one the client's database with our own.
 	fn replace_client_db(&self) -> Result<(), Error> {
 		let our_db = self.restoration_db();
@@ -406,10 +414,27 @@ impl Service {
 	/// Initialize the restoration synchronously.
 	/// The recover flag indicates whether to recover the restored snapshot.
 	pub fn init_restore(&self, manifest: ManifestData, recover: bool) -> Result<(), Error> {
-		let rest_dir = self.restoration_dir();
-
 		let mut res = self.restoration.lock();
 
+		let rest_dir = self.restoration_dir();
+		let rest_db = self.restoration_db();
+		let recovery_temp = self.temp_recovery_dir();
+		let prev_chunks = self.prev_chunks_dir();
+
+		// delete and restore the restoration dir.
+		if let Err(e) = fs::remove_dir_all(&prev_chunks) {
+			match e.kind() {
+				ErrorKind::NotFound => {},
+				_ => return Err(e.into()),
+			}
+		}
+
+		// Move the previous recovery temp directory
+		// to `prev_chunks` to be able to restart restoring
+		// with previously downloaded blocks
+		// This step is optional, so don't fail on error
+		fs::rename(&recovery_temp, &prev_chunks).ok();
+
 		self.state_chunks.store(0, Ordering::SeqCst);
 		self.block_chunks.store(0, Ordering::SeqCst);
 
@@ -424,29 +449,38 @@ impl Service {
 			}
 		}
 
+		*self.status.lock() = RestorationStatus::Initializing {
+			chunks_done: 0,
+		};
+
 		fs::create_dir_all(&rest_dir)?;
 
 		// make new restoration.
 		let writer = match recover {
-			true => Some(LooseWriter::new(self.temp_recovery_dir())?),
+			true => Some(LooseWriter::new(recovery_temp)?),
 			false => None
 		};
 
 		let params = RestorationParams {
-			manifest: manifest,
+			manifest: manifest.clone(),
 			pruning: self.pruning,
-			db: self.restoration_db_handler.open(&self.restoration_db())?,
+			db: self.restoration_db_handler.open(&rest_db)?,
 			writer: writer,
 			genesis: &self.genesis_block,
-			guard: Guard::new(rest_dir),
+			guard: Guard::new(rest_db),
 			engine: &*self.engine,
 		};
 
-		let state_chunks = params.manifest.state_hashes.len();
-		let block_chunks = params.manifest.block_hashes.len();
+		let state_chunks = manifest.state_hashes.len();
+		let block_chunks = manifest.block_hashes.len();
 
 		*res = Some(Restoration::new(params)?);
 
+		self.restoring_snapshot.store(true, Ordering::SeqCst);
+
+		// Import previous chunks, continue if it fails
+		self.import_prev_chunks(&mut res, manifest).ok();
+
 		*self.status.lock() = RestorationStatus::Ongoing {
 			state_chunks: state_chunks as u32,
 			block_chunks: block_chunks as u32,
@@ -454,10 +488,65 @@ impl Service {
 			block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32,
 		};
 
-		self.restoring_snapshot.store(true, Ordering::SeqCst);
 		Ok(())
 	}
 
+	/// Import the previous chunks into the current restoration
+	fn import_prev_chunks(&self, restoration: &mut Option<Restoration>, manifest: ManifestData) -> Result<(), Error> {
+		let prev_chunks = self.prev_chunks_dir();
+
+		// Restore previous snapshot chunks
+		let files = fs::read_dir(prev_chunks.as_path())?;
+		let mut num_temp_chunks = 0;
+
+		for prev_chunk_file in files {
+			// Don't go over all the files if the restoration has been aborted
+			if !self.restoring_snapshot.load(Ordering::SeqCst) {
+				trace!(target:"snapshot", "Aborting importing previous chunks");
+				return Ok(());
+			}
+			// Import the chunk, don't fail and continue if one fails
+			match self.import_prev_chunk(restoration, &manifest, prev_chunk_file) {
+				Ok(true) => num_temp_chunks += 1,
+				Err(e) => trace!(target: "snapshot", "Error importing chunk: {:?}", e),
+				_ => (),
+			}
+		}
+
+		trace!(target:"snapshot", "Imported {} previous chunks", num_temp_chunks);
+
+		// Remove the prev temp directory
+		fs::remove_dir_all(&prev_chunks)?;
+
+		Ok(())
+	}
+
+	/// Import a previous chunk at the given path. Returns whether the block was imported or not
+	fn import_prev_chunk(&self, restoration: &mut Option<Restoration>, manifest: &ManifestData, file: io::Result<fs::DirEntry>) -> Result<bool, Error> {
+		let file = file?;
+		let path = file.path();
+
+		let mut file = File::open(path.clone())?;
+		let mut buffer = Vec::new();
+		file.read_to_end(&mut buffer)?;
+
+		let hash = keccak(&buffer);
+
+		let is_state = if manifest.block_hashes.contains(&hash) {
+			false
+		} else if manifest.state_hashes.contains(&hash) {
+			true
+		} else {
+			return Ok(false);
+		};
+
+		self.feed_chunk_with_restoration(restoration, hash, &buffer, is_state)?;
+
+		trace!(target: "snapshot", "Fed chunk {:?}", hash);
+
+		Ok(true)
+	}
+
 	// finalize the restoration. this accepts an already-locked
 	// restoration as an argument -- so acquiring it again _will_
 	// lead to deadlock.
@@ -499,12 +588,19 @@ impl Service {
 	/// Feed a chunk of either kind. no-op if no restoration or status is wrong.
 	fn feed_chunk(&self, hash: H256, chunk: &[u8], is_state: bool) -> Result<(), Error> {
 		// TODO: be able to process block chunks and state chunks at same time?
-		let (result, db) = {
-			let mut restoration = self.restoration.lock();
+		let mut restoration = self.restoration.lock();
+		self.feed_chunk_with_restoration(&mut restoration, hash, chunk, is_state)
+	}
 
+	/// Feed a chunk with the Restoration
+	fn feed_chunk_with_restoration(&self, restoration: &mut Option<Restoration>, hash: H256, chunk: &[u8], is_state: bool) -> Result<(), Error> {
+		let (result, db) = {
 			match self.status() {
-				RestorationStatus::Inactive | RestorationStatus::Failed => return Ok(()),
-				RestorationStatus::Ongoing { .. } => {
+				RestorationStatus::Inactive | RestorationStatus::Failed => {
+					trace!(target: "snapshot", "Tried to restore chunk {:x} while inactive or failed", hash);
+					return Ok(());
+				},
+				RestorationStatus::Ongoing { .. } | RestorationStatus::Initializing { .. } => {
 					let (res, db) = {
 						let rest = match *restoration {
 							Some(ref mut r) => r,
@@ -583,11 +679,41 @@ impl SnapshotService for Service {
 		self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok())
 	}
 
+	fn completed_chunks(&self) -> Option<Vec<H256>> {
+		let restoration = self.restoration.lock();
+
+		match *restoration {
+			Some(ref restoration) => {
+				let completed_chunks = restoration.manifest.block_hashes
+					.iter()
+					.filter(|h| !restoration.block_chunks_left.contains(h))
+					.chain(
+						restoration.manifest.state_hashes
+							.iter()
+							.filter(|h| !restoration.state_chunks_left.contains(h))
+					)
+					.map(|h| *h)
+					.collect();
+
+				Some(completed_chunks)
+			},
+			None => None,
+		}
+	}
+
 	fn status(&self) -> RestorationStatus {
 		let mut cur_status = self.status.lock();
-		if let RestorationStatus::Ongoing { ref mut state_chunks_done, ref mut block_chunks_done, .. } = *cur_status {
-			*state_chunks_done = self.state_chunks.load(Ordering::SeqCst) as u32;
-			*block_chunks_done = self.block_chunks.load(Ordering::SeqCst) as u32;
+
+		match *cur_status {
+			RestorationStatus::Initializing { ref mut chunks_done } => {
+				*chunks_done = self.state_chunks.load(Ordering::SeqCst) as u32 +
+					self.block_chunks.load(Ordering::SeqCst) as u32;
+			}
+			RestorationStatus::Ongoing { ref mut state_chunks_done, ref mut block_chunks_done, .. } => {
+				*state_chunks_done = self.state_chunks.load(Ordering::SeqCst) as u32;
+				*block_chunks_done = self.block_chunks.load(Ordering::SeqCst) as u32;
+			},
+			_ => (),
 		}
 
 		cur_status.clone()
@@ -600,6 +726,7 @@ impl SnapshotService for Service {
 	}
 
 	fn abort_restore(&self) {
+		trace!(target: "snapshot", "Aborting restore");
 		self.restoring_snapshot.store(false, Ordering::SeqCst);
 		*self.restoration.lock() = None;
 		*self.status.lock() = RestorationStatus::Inactive;
@@ -616,6 +743,10 @@ impl SnapshotService for Service {
 			trace!("Error sending snapshot service message: {:?}", e);
 		}
 	}
+
+	fn shutdown(&self) {
+		self.abort_restore();
+	}
 }
 
 impl Drop for Service {

ethcore/src/snapshot/tests/helpers.rs

@@ -75,7 +75,7 @@ impl StateProducer {
 
 		// sweep once to alter storage tries.
 		for &mut (ref mut address_hash, ref mut account_data) in &mut accounts_to_modify {
-			let mut account: BasicAccount = ::rlp::decode(&*account_data);
+			let mut account: BasicAccount = ::rlp::decode(&*account_data).expect("error decoding basic account");
 			let acct_db = AccountDBMut::from_hash(db, *address_hash);
 			fill_storage(acct_db, &mut account.storage_root, &mut self.storage_seed);
 			*account_data = DBValue::from_vec(::rlp::encode(&account).into_vec());

ethcore/src/snapshot/tests/proof_of_authority.rs

@@ -215,7 +215,7 @@ fn fixed_to_contract_only() {
 		secret!("dog42"),
 	]);
 
-	assert!(provider.has_account(*RICH_ADDR).unwrap());
+	assert!(provider.has_account(*RICH_ADDR));
 
 	let client = make_chain(provider, 3, vec![
 		Transition::Manual(3, vec![addrs[2], addrs[3], addrs[5], addrs[7]]),
@@ -248,7 +248,7 @@ fn fixed_to_contract_to_contract() {
 		secret!("dog42"),
 	]);
 
-	assert!(provider.has_account(*RICH_ADDR).unwrap());
+	assert!(provider.has_account(*RICH_ADDR));
 
 	let client = make_chain(provider, 3, vec![
 		Transition::Manual(3, vec![addrs[2], addrs[3], addrs[5], addrs[7]]),

ethcore/src/snapshot/tests/service.rs

@@ -130,12 +130,16 @@ fn guards_delete_folders() {
 	service.init_restore(manifest.clone(), true).unwrap();
 	assert!(path.exists());
 
+	// The `db` folder should have been deleted,
+	// while the `temp` one kept
 	service.abort_restore();
-	assert!(!path.exists());
+	assert!(!path.join("db").exists());
+	assert!(path.join("temp").exists());
 
 	service.init_restore(manifest.clone(), true).unwrap();
 	assert!(path.exists());
 
 	drop(service);
-	assert!(!path.exists());
+	assert!(!path.join("db").exists());
+	assert!(path.join("temp").exists());
 }

ethcore/src/snapshot/tests/state.rs

@@ -114,7 +114,7 @@ fn get_code_from_prev_chunk() {
 	// first one will have code inlined,
 	// second will just have its hash.
 	let thin_rlp = acc_stream.out();
-	let acc: BasicAccount = ::rlp::decode(&thin_rlp);
+	let acc: BasicAccount = ::rlp::decode(&thin_rlp).expect("error decoding basic account");
 
 	let mut make_chunk = |acc, hash| {
 		let mut db = MemoryDB::new();

ethcore/src/snapshot/traits.rs

@@ -30,6 +30,9 @@ pub trait SnapshotService : Sync + Send {
 	/// `None` indicates warp sync isn't supported by the consensus engine.
 	fn supported_versions(&self) -> Option<(u64, u64)>;
 
+	/// Returns a list of the completed chunks
+	fn completed_chunks(&self) -> Option<Vec<H256>>;
+
 	/// Get raw chunk for a given hash.
 	fn chunk(&self, hash: H256) -> Option<Bytes>;
 
@@ -51,4 +54,7 @@ pub trait SnapshotService : Sync + Send {
 	/// Feed a raw block chunk to the service to be processed asynchronously.
 	/// no-op if currently restoring.
 	fn restore_block_chunk(&self, hash: H256, chunk: Bytes);
+
+	/// Shutdown the Snapshot Service by aborting any ongoing restore
+	fn shutdown(&self);
 }

ethcore/src/snapshot/watcher.rs

@@ -17,14 +17,14 @@
 //! Watcher for snapshot-related chain events.
 
 use parking_lot::Mutex;
-use client::{BlockInfo, Client, ChainNotify, ClientIoMessage};
+use client::{BlockInfo, Client, ChainNotify, ChainRoute, ClientIoMessage};
 use ids::BlockId;
 
 use io::IoChannel;
 use ethereum_types::H256;
 use bytes::Bytes;
 
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
 // helper trait for transforming hashes to numbers and checking if syncing.
 trait Oracle: Send + Sync {
@@ -103,11 +103,10 @@ impl ChainNotify for Watcher {
 		&self,
 		imported: Vec<H256>,
 		_: Vec<H256>,
-		_: Vec<H256>,
-		_: Vec<H256>,
+		_: ChainRoute,
 		_: Vec<H256>,
 		_: Vec<Bytes>,
-		_duration: u64)
+		_duration: Duration)
 	{
 		if self.oracle.is_major_importing() { return }
 
@@ -131,11 +130,12 @@ impl ChainNotify for Watcher {
 mod tests {
 	use super::{Broadcast, Oracle, Watcher};
 
-	use client::ChainNotify;
+	use client::{ChainNotify, ChainRoute};
 
 	use ethereum_types::{H256, U256};
 
 	use std::collections::HashMap;
+	use std::time::Duration;
 
 	struct TestOracle(HashMap<H256, u64>);
 
@@ -158,6 +158,8 @@ mod tests {
 
 	// helper harness for tests which expect a notification.
 	fn harness(numbers: Vec<u64>, period: u64, history: u64, expected: Option<u64>) {
+		const DURATION_ZERO: Duration = Duration::from_millis(0);
+
 		let hashes: Vec<_> = numbers.clone().into_iter().map(|x| H256::from(U256::from(x))).collect();
 		let map = hashes.clone().into_iter().zip(numbers).collect();
 
@@ -171,11 +173,10 @@ mod tests {
 		watcher.new_blocks(
 			hashes,
 			vec![],
+			ChainRoute::default(),
 			vec![],
 			vec![],
-			vec![],
-			vec![],
-			0,
+			DURATION_ZERO,
 		);
 	}
 

ethcore/src/spec/spec.rs

@@ -48,6 +48,8 @@ use trace::{NoopTracer, NoopVMTracer};
 
 pub use ethash::OptimizeFor;
 
+const MAX_TRANSACTION_SIZE: usize = 300 * 1024;
+
 // helper for formatting errors.
 fn fmt_err<F: ::std::fmt::Display>(f: F) -> String {
 	format!("Spec json is invalid: {}", f)
@@ -123,6 +125,8 @@ pub struct CommonParams {
 	pub max_code_size_transition: BlockNumber,
 	/// Transaction permission managing contract address.
 	pub transaction_permission_contract: Option<Address>,
+	/// Maximum size of transaction's RLP payload
+	pub max_transaction_size: usize,
 }
 
 impl CommonParams {
@@ -238,6 +242,7 @@ impl From<ethjson::spec::Params> for CommonParams {
 			registrar: p.registrar.map_or_else(Address::new, Into::into),
 			node_permission_contract: p.node_permission_contract.map(Into::into),
 			max_code_size: p.max_code_size.map_or(u64::max_value(), Into::into),
+			max_transaction_size: p.max_transaction_size.map_or(MAX_TRANSACTION_SIZE, Into::into),
 			max_code_size_transition: p.max_code_size_transition.map_or(0, Into::into),
 			transaction_permission_contract: p.transaction_permission_contract.map(Into::into),
 			wasm_activation_transition: p.wasm_activation_transition.map_or(
@@ -771,7 +776,7 @@ impl Spec {
 				author: *genesis.author(),
 				timestamp: genesis.timestamp(),
 				difficulty: *genesis.difficulty(),
-				gas_limit: *genesis.gas_limit(),
+				gas_limit: U256::max_value(),
 				last_hashes: Arc::new(Vec::new()),
 				gas_used: 0.into(),
 			};
@@ -780,7 +785,7 @@ impl Spec {
 			let tx = Transaction {
 				nonce: self.engine.account_start_nonce(0),
 				action: Action::Call(a),
-				gas: U256::from(50_000_000), // TODO: share with client.
+				gas: U256::max_value(),
 				gas_price: U256::default(),
 				value: U256::default(),
 				data: d,

ethcore/src/state/account.rs

@@ -21,6 +21,7 @@ use std::sync::Arc;
 use std::collections::{HashMap, BTreeMap};
 use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak};
 use ethereum_types::{H256, U256, Address};
+use error::Error;
 use hashdb::HashDB;
 use kvdb::DBValue;
 use bytes::{Bytes, ToPretty};
@@ -144,9 +145,10 @@ impl Account {
 	}
 
 	/// Create a new account from RLP.
-	pub fn from_rlp(rlp: &[u8]) -> Account {
-		let basic: BasicAccount = ::rlp::decode(rlp);
-		basic.into()
+	pub fn from_rlp(rlp: &[u8]) -> Result<Account, Error> {
+		::rlp::decode::<BasicAccount>(rlp)
+			.map(|ba| ba.into())
+			.map_err(|e| e.into())
 	}
 
 	/// Create a new contract account.
@@ -202,8 +204,8 @@ impl Account {
 			return Ok(value);
 		}
 		let db = SecTrieDB::new(db, &self.storage_root)?;
-
-		let item: U256 = db.get_with(key, ::rlp::decode)?.unwrap_or_else(U256::zero);
+		let panicky_decoder = |bytes:&[u8]| ::rlp::decode(&bytes).expect("decoding db value failed");
+		let item: U256 = db.get_with(key, panicky_decoder)?.unwrap_or_else(U256::zero);
 		let value: H256 = item.into();
 		self.storage_cache.borrow_mut().insert(key.clone(), value.clone());
 		Ok(value)
@@ -478,7 +480,8 @@ impl Account {
 
 		let trie = TrieDB::new(db, &self.storage_root)?;
 		let item: U256 = {
-			let query = (&mut recorder, ::rlp::decode);
+			let panicky_decoder = |bytes:&[u8]| ::rlp::decode(bytes).expect("decoding db value failed");
+			let query = (&mut recorder, panicky_decoder);
 			trie.get_with(&storage_key, query)?.unwrap_or_else(U256::zero)
 		};
 
@@ -528,7 +531,7 @@ mod tests {
 			a.rlp()
 		};
 
-		let a = Account::from_rlp(&rlp);
+		let a = Account::from_rlp(&rlp).expect("decoding db value failed");
 		assert_eq!(*a.storage_root().unwrap(), "c57e1afb758b07f8d2c8f13a3b6e44fa5ff94ab266facc5a4fd3f062426e50b2".into());
 		assert_eq!(a.storage_at(&db.immutable(), &0x00u64.into()).unwrap(), 0x1234u64.into());
 		assert_eq!(a.storage_at(&db.immutable(), &0x01u64.into()).unwrap(), H256::default());
@@ -546,10 +549,10 @@ mod tests {
 			a.rlp()
 		};
 
-		let mut a = Account::from_rlp(&rlp);
+		let mut a = Account::from_rlp(&rlp).expect("decoding db value failed");
 		assert!(a.cache_code(&db.immutable()).is_some());
 
-		let mut a = Account::from_rlp(&rlp);
+		let mut a = Account::from_rlp(&rlp).expect("decoding db value failed");
 		assert_eq!(a.note_code(vec![0x55, 0x44, 0xffu8]), Ok(()));
 	}
 
@@ -609,7 +612,7 @@ mod tests {
 	#[test]
 	fn rlpio() {
 		let a = Account::new(69u8.into(), 0u8.into(), HashMap::new(), Bytes::new());
-		let b = Account::from_rlp(&a.rlp());
+		let b = Account::from_rlp(&a.rlp()).unwrap();
 		assert_eq!(a.balance(), b.balance());
 		assert_eq!(a.nonce(), b.nonce());
 		assert_eq!(a.code_hash(), b.code_hash());

ethcore/src/state/mod.rs

@@ -21,7 +21,7 @@
 
 use std::cell::{RefCell, RefMut};
 use std::collections::hash_map::Entry;
-use std::collections::{HashMap, BTreeMap, HashSet};
+use std::collections::{HashMap, BTreeMap, BTreeSet, HashSet};
 use std::fmt;
 use std::sync::Arc;
 use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY};
@@ -605,7 +605,8 @@ impl<B: Backend> State<B> {
 
 		// account is not found in the global cache, get from the DB and insert into local
 		let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
-		let maybe_acc = db.get_with(address, Account::from_rlp)?;
+		let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed");
+		let maybe_acc = db.get_with(address, from_rlp)?;
 		let r = maybe_acc.as_ref().map_or(Ok(H256::new()), |a| {
 			let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), a.address_hash(address));
 			a.storage_at(account_db.as_hashdb(), key)
@@ -862,40 +863,65 @@ impl<B: Backend> State<B> {
 		}))
 	}
 
-	// Return a list of all touched addresses in cache.
-	fn touched_addresses(&self) -> Vec<Address> {
+	/// Populate a PodAccount map from this state, with another state as the account and storage query.
+	pub fn to_pod_diff<X: Backend>(&mut self, query: &State<X>) -> trie::Result<PodState> {
 		assert!(self.checkpoints.borrow().is_empty());
-		self.cache.borrow().iter().map(|(add, _)| *add).collect()
-	}
 
-	fn query_pod(&mut self, query: &PodState, touched_addresses: &[Address]) -> trie::Result<()> {
-		let pod = query.get();
+		// Merge PodAccount::to_pod for cache of self and `query`.
+		let all_addresses = self.cache.borrow().keys().cloned()
+			.chain(query.cache.borrow().keys().cloned())
+			.collect::<BTreeSet<_>>();
 
-		for address in touched_addresses {
-			if !self.ensure_cached(address, RequireCache::Code, true, |a| a.is_some())? {
-				continue
+		Ok(PodState::from(all_addresses.into_iter().fold(Ok(BTreeMap::new()), |m: trie::Result<_>, address| {
+			let mut m = m?;
+
+			let account = self.ensure_cached(&address, RequireCache::Code, true, |acc| {
+				acc.map(|acc| {
+					// Merge all modified storage keys.
+					let all_keys = {
+						let self_keys = acc.storage_changes().keys().cloned()
+							.collect::<BTreeSet<_>>();
+
+						if let Some(ref query_storage) = query.cache.borrow().get(&address)
+							.and_then(|opt| {
+								Some(opt.account.as_ref()?.storage_changes().keys().cloned()
+									 .collect::<BTreeSet<_>>())
+							})
+						{
+							self_keys.union(&query_storage).cloned().collect::<Vec<_>>()
+						} else {
+							self_keys.into_iter().collect::<Vec<_>>()
+						}
+					};
+
+					// Storage must be fetched after ensure_cached to avoid borrow problem.
+					(*acc.balance(), *acc.nonce(), all_keys, acc.code().map(|x| x.to_vec()))
+				})
+			})?;
+
+			if let Some((balance, nonce, storage_keys, code)) = account {
+				let storage = storage_keys.into_iter().fold(Ok(BTreeMap::new()), |s: trie::Result<_>, key| {
+					let mut s = s?;
+
+					s.insert(key, self.storage_at(&address, &key)?);
+					Ok(s)
+				})?;
+
+				m.insert(address, PodAccount {
+					balance, nonce, storage, code
+				});
 			}
 
-			if let Some(pod_account) = pod.get(address) {
-				// needs to be split into two parts for the refcell code here
-				// to work.
-				for key in pod_account.storage.keys() {
-					self.storage_at(address, key)?;
-				}
-			}
-		}
-
-		Ok(())
+			Ok(m)
+		})?))
 	}
 
 	/// Returns a `StateDiff` describing the difference from `orig` to `self`.
 	/// Consumes self.
-	pub fn diff_from<X: Backend>(&self, orig: State<X>) -> trie::Result<StateDiff> {
-		let addresses_post = self.touched_addresses();
+	pub fn diff_from<X: Backend>(&self, mut orig: State<X>) -> trie::Result<StateDiff> {
 		let pod_state_post = self.to_pod();
-		let mut state_pre = orig;
-		state_pre.query_pod(&pod_state_post, &addresses_post)?;
-		Ok(pod_state::diff_pod(&state_pre.to_pod(), &pod_state_post))
+		let pod_state_pre = orig.to_pod_diff(self)?;
+		Ok(pod_state::diff_pod(&pod_state_pre, &pod_state_post))
 	}
 
 	// load required account data from the databases.
@@ -958,7 +984,8 @@ impl<B: Backend> State<B> {
 
 				// not found in the global cache, get from the DB and insert into local
 				let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root)?;
-				let mut maybe_acc = db.get_with(a, Account::from_rlp)?;
+				let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed");
+				let mut maybe_acc = db.get_with(a, from_rlp)?;
 				if let Some(ref mut account) = maybe_acc.as_mut() {
 					let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a));
 					Self::update_account_cache(require, account, &self.db, accountdb.as_hashdb());
@@ -987,7 +1014,8 @@ impl<B: Backend> State<B> {
 				None => {
 					let maybe_acc = if !self.db.is_known_null(a) {
 						let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root)?;
-						AccountEntry::new_clean(db.get_with(a, Account::from_rlp)?)
+						let from_rlp = |b:&[u8]| { Account::from_rlp(b).expect("decoding db value failed") };
+						AccountEntry::new_clean(db.get_with(a, from_rlp)?)
 					} else {
 						AccountEntry::new_clean(None)
 					};
@@ -1039,7 +1067,10 @@ impl<B: Backend> State<B> {
 		let mut recorder = Recorder::new();
 		let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
 		let maybe_account: Option<BasicAccount> = {
-			let query = (&mut recorder, ::rlp::decode);
+			let panicky_decoder = |bytes: &[u8]| {
+				::rlp::decode(bytes).expect(&format!("prove_account, could not query trie for account key={}", &account_key))
+			};
+			let query = (&mut recorder, panicky_decoder);
 			trie.get_with(&account_key, query)?
 		};
 		let account = maybe_account.unwrap_or_else(|| BasicAccount {
@@ -1061,7 +1092,8 @@ impl<B: Backend> State<B> {
 		// TODO: probably could look into cache somehow but it's keyed by
 		// address, not keccak(address).
 		let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
-		let acc = match trie.get_with(&account_key, Account::from_rlp)? {
+		let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed");
+		let acc = match trie.get_with(&account_key, from_rlp)? {
 			Some(acc) => acc,
 			None => return Ok((Vec::new(), H256::new())),
 		};
@@ -2243,9 +2275,6 @@ mod tests {
 		let original = state.clone();
 		state.kill_account(&a);
 
-		assert_eq!(original.touched_addresses(), vec![]);
-		assert_eq!(state.touched_addresses(), vec![a]);
-
 		let diff = state.diff_from(original).unwrap();
 		let diff_map = diff.get();
 		assert_eq!(diff_map.len(), 1);
@@ -2258,4 +2287,42 @@ mod tests {
 					   storage: Default::default()
 				   }), None).as_ref());
 	}
+
+	#[test]
+	fn should_trace_diff_unmodified_storage() {
+		use pod_account;
+
+		let a = 10.into();
+		let db = get_temp_state_db();
+
+		let (root, db) = {
+			let mut state = State::new(db, U256::from(0), Default::default());
+			state.set_storage(&a, H256::from(&U256::from(1u64)), H256::from(&U256::from(20u64))).unwrap();
+			state.commit().unwrap();
+			state.drop()
+		};
+
+		let mut state = State::from_existing(db, root, U256::from(0u8), Default::default()).unwrap();
+		let original = state.clone();
+		state.set_storage(&a, H256::from(&U256::from(1u64)), H256::from(&U256::from(100u64))).unwrap();
+
+		let diff = state.diff_from(original).unwrap();
+		let diff_map = diff.get();
+		assert_eq!(diff_map.len(), 1);
+		assert!(diff_map.get(&a).is_some());
+		assert_eq!(diff_map.get(&a),
+				   pod_account::diff_pod(Some(&PodAccount {
+					   balance: U256::zero(),
+					   nonce: U256::zero(),
+					   code: Some(Default::default()),
+					   storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(20u64)))]
+						   .into_iter().collect(),
+				   }), Some(&PodAccount {
+					   balance: U256::zero(),
+					   nonce: U256::zero(),
+					   code: Some(Default::default()),
+					   storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(100u64)))]
+						   .into_iter().collect(),
+				   })).as_ref());
+	}
 }

ethcore/src/tests/client.rs

@@ -36,6 +36,7 @@ use views::BlockView;
 use ethkey::KeyPair;
 use transaction::{PendingTransaction, Transaction, Action, Condition};
 use miner::MinerService;
+use rlp::{RlpStream, EMPTY_LIST_RLP};
 use tempdir::TempDir;
 
 #[test]
@@ -111,6 +112,25 @@ fn imports_good_block() {
 	assert!(!block.into_inner().is_empty());
 }
 
+#[test]
+fn fails_to_import_block_with_invalid_rlp() {
+	use error::{BlockImportError, BlockImportErrorKind};
+
+	let client = generate_dummy_client(6);
+	let mut rlp = RlpStream::new_list(3);
+	rlp.append_raw(&EMPTY_LIST_RLP, 1); // empty header
+	rlp.append_raw(&EMPTY_LIST_RLP, 1);
+	rlp.append_raw(&EMPTY_LIST_RLP, 1);
+	let invalid_header_block = rlp.out();
+
+	match client.import_block(invalid_header_block) {
+		Err(BlockImportError(BlockImportErrorKind::Decoder(_), _)) => (), // all good
+		Err(_) => panic!("Should fail with a decoder error"),
+		Ok(_) => panic!("Should not import block with invalid header"),
+	}
+}
+
+
 #[test]
 fn query_none_block() {
 	let tempdir = TempDir::new("").unwrap();

ethcore/src/trace/types/flat.rs

@@ -244,7 +244,7 @@ mod tests {
 		]);
 
 		let encoded = ::rlp::encode(&block_traces);
-		let decoded = ::rlp::decode(&encoded);
+		let decoded = ::rlp::decode(&encoded).expect("error decoding block traces");
 		assert_eq!(block_traces, decoded);
 	}
 }

ethcore/src/verification/queue/kind.rs

@@ -119,14 +119,13 @@ pub mod blocks {
 
 	impl Unverified {
 		/// Create an `Unverified` from raw bytes.
-		pub fn new(bytes: Bytes) -> Self {
-			use views::BlockView;
+		pub fn from_rlp(bytes: Bytes) -> Result<Self, ::rlp::DecoderError> {
 
-			let header = view!(BlockView, &bytes).header();
-			Unverified {
+			let header = ::rlp::Rlp::new(&bytes).val_at(0)?; 
+			Ok(Unverified {
 				header: header,
 				bytes: bytes,
-			}
+			})
 		}
 	}
 

ethcore/src/verification/queue/mod.rs

@@ -734,6 +734,7 @@ mod tests {
 	use test_helpers::{get_good_dummy_block_seq, get_good_dummy_block};
 	use error::*;
 	use views::BlockView;
+	use bytes::Bytes;
 
 	// create a test block queue.
 	// auto_scaling enables verifier adjustment.
@@ -746,6 +747,10 @@ mod tests {
 		BlockQueue::new(config, engine, IoChannel::disconnected(), true)
 	}
 
+	fn new_unverified(bytes: Bytes) -> Unverified {
+		Unverified::from_rlp(bytes).expect("Should be valid rlp")
+	}
+
 	#[test]
 	fn can_be_created() {
 		// TODO better test
@@ -757,7 +762,7 @@ mod tests {
 	#[test]
 	fn can_import_blocks() {
 		let queue = get_test_queue(false);
-		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
+		if let Err(e) = queue.import(new_unverified(get_good_dummy_block())) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 	}
@@ -765,11 +770,11 @@ mod tests {
 	#[test]
 	fn returns_error_for_duplicates() {
 		let queue = get_test_queue(false);
-		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
+		if let Err(e) = queue.import(new_unverified(get_good_dummy_block())) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 
-		let duplicate_import = queue.import(Unverified::new(get_good_dummy_block()));
+		let duplicate_import = queue.import(new_unverified(get_good_dummy_block()));
 		match duplicate_import {
 			Err(e) => {
 				match e {
@@ -786,7 +791,7 @@ mod tests {
 		let queue = get_test_queue(false);
 		let block = get_good_dummy_block();
 		let hash = view!(BlockView, &block).header().hash().clone();
-		if let Err(e) = queue.import(Unverified::new(block)) {
+		if let Err(e) = queue.import(new_unverified(block)) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 		queue.flush();
@@ -802,14 +807,14 @@ mod tests {
 		let queue = get_test_queue(false);
 		let block = get_good_dummy_block();
 		let hash = view!(BlockView, &block).header().hash().clone();
-		if let Err(e) = queue.import(Unverified::new(block)) {
+		if let Err(e) = queue.import(new_unverified(block)) {
 			panic!("error importing block that is valid by definition({:?})", e);
 		}
 		queue.flush();
 		queue.drain(10);
 		queue.mark_as_good(&[ hash ]);
 
-		if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) {
+		if let Err(e) = queue.import(new_unverified(get_good_dummy_block())) {
 			panic!("error importing block that has already been drained ({:?})", e);
 		}
 	}
@@ -817,7 +822,7 @@ mod tests {
 	#[test]
 	fn returns_empty_once_finished() {
 		let queue = get_test_queue(false);
-		queue.import(Unverified::new(get_good_dummy_block()))
+		queue.import(new_unverified(get_good_dummy_block()))
 			.expect("error importing block that is valid by definition");
 		queue.flush();
 		queue.drain(1);
@@ -835,7 +840,7 @@ mod tests {
 		assert!(!queue.queue_info().is_full());
 		let mut blocks = get_good_dummy_block_seq(50);
 		for b in blocks.drain(..) {
-			queue.import(Unverified::new(b)).unwrap();
+			queue.import(new_unverified(b)).unwrap();
 		}
 		assert!(queue.queue_info().is_full());
 	}
@@ -863,7 +868,7 @@ mod tests {
 		*queue.state.0.lock() = State::Work(0);
 
 		for block in get_good_dummy_block_seq(5000) {
-			queue.import(Unverified::new(block)).expect("Block good by definition; qed");
+			queue.import(new_unverified(block)).expect("Block good by definition; qed");
 		}
 
 		// almost all unverified == bump verifier count.

ethcore/src/verification/verification.rs

@@ -224,7 +224,7 @@ fn verify_uncles(header: &Header, bytes: &[u8], bc: &BlockProvider, engine: &Eth
 				return Err(From::from(BlockError::UncleParentNotInChain(uncle_parent.hash())));
 			}
 
-			let uncle_parent = uncle_parent.decode();
+			let uncle_parent = uncle_parent.decode()?;
 			verify_parent(&uncle, &uncle_parent, engine)?;
 			engine.verify_block_family(&uncle, &uncle_parent)?;
 			verified.insert(uncle.hash());
@@ -476,7 +476,7 @@ mod tests {
 			unimplemented!()
 		}
 
-		fn logs<F>(&self, _blocks: Vec<BlockNumber>, _matches: F, _limit: Option<usize>) -> Vec<LocalizedLogEntry>
+		fn logs<F>(&self, _blocks: Vec<H256>, _matches: F, _limit: Option<usize>) -> Vec<LocalizedLogEntry>
 			where F: Fn(&LogEntry) -> bool, Self: Sized {
 			unimplemented!()
 		}
@@ -500,10 +500,9 @@ mod tests {
 		// no existing tests need access to test, so having this not function
 		// is fine.
 		let client = ::client::TestBlockChainClient::default();
-
 		let parent = bc.block_header_data(header.parent_hash())
 			.ok_or(BlockError::UnknownParent(header.parent_hash().clone()))?
-			.decode();
+			.decode()?;
 
 		let full_params = FullFamilyParams {
 			block_bytes: bytes,
@@ -575,7 +574,17 @@ mod tests {
 			nonce: U256::from(2)
 		}.sign(keypair.secret(), None);
 
+		let tr3 = Transaction {
+			action: Action::Call(0x0.into()),
+			value: U256::from(0),
+			data: Bytes::new(),
+			gas: U256::from(30_000),
+			gas_price: U256::from(0),
+			nonce: U256::zero(),
+		}.null_sign(0);
+
 		let good_transactions = [ tr1.clone(), tr2.clone() ];
+		let eip86_transactions = [ tr3.clone() ];
 
 		let diff_inc = U256::from(0x40);
 
@@ -611,6 +620,7 @@ mod tests {
 		uncles_rlp.append_list(&good_uncles);
 		let good_uncles_hash = keccak(uncles_rlp.as_raw());
 		let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<UnverifiedTransaction>(t)));
+		let eip86_transactions_root = ordered_trie_root(eip86_transactions.iter().map(|t| ::rlp::encode::<UnverifiedTransaction>(t)));
 
 		let mut parent = good.clone();
 		parent.set_number(9);
@@ -631,6 +641,14 @@ mod tests {
 
 		check_ok(basic_test(&create_test_block(&good), engine));
 
+		let mut bad_header = good.clone();
+		bad_header.set_transactions_root(eip86_transactions_root.clone());
+		bad_header.set_uncles_hash(good_uncles_hash.clone());
+		match basic_test(&create_test_block_with_data(&bad_header, &eip86_transactions, &good_uncles), engine) {
+			Err(Error(ErrorKind::Transaction(ref e), _)) if e == &::ethkey::Error::InvalidSignature.into() => (),
+			e => panic!("Block verification failed.\nExpected: Transaction Error (Invalid Signature)\nGot: {:?}", e),
+		}
+
 		let mut header = good.clone();
 		header.set_transactions_root(good_transactions_root.clone());
 		header.set_uncles_hash(good_uncles_hash.clone());

ethcore/src/views/block.rs

@@ -29,7 +29,6 @@ pub struct BlockView<'a> {
 	rlp: ViewRlp<'a>
 }
 
-
 impl<'a> BlockView<'a> {
 	/// Creates new view onto block from rlp.
 	/// Use the `view!` macro to create this view in order to capture debugging info.
@@ -39,9 +38,9 @@ impl<'a> BlockView<'a> {
 	/// ```
 	/// #[macro_use]
 	/// extern crate ethcore;
-	/// 
+	///
 	/// use ethcore::views::{BlockView};
-	/// 
+	///
 	/// fn main() {
 	/// let bytes : &[u8] = &[];
 	/// let block_view = view!(BlockView, bytes);

ethcore/src/views/view_rlp.rs

@@ -39,10 +39,10 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view {
 
 	/// Returns a new instance replacing existing rlp with new rlp, maintaining debug info
 	fn new_from_rlp(&self, rlp: Rlp<'a>) -> Self {
-		ViewRlp { 
-			rlp, 
+		ViewRlp {
+			rlp,
 			file: self.file,
-			line: self.line 
+			line: self.line
 		}
 	}
 
@@ -53,7 +53,12 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view {
 	}
 
 	fn expect_valid_rlp<T>(&self, r: Result<T, DecoderError>) -> T {
-		r.expect(&format!("View rlp is trusted and should be valid. Constructed in {} on line {}", self.file, self.line))
+		r.unwrap_or_else(|e| panic!(
+			"View rlp is trusted and should be valid. Constructed in {} on line {}: {}",
+			self.file,
+			self.line,
+			e
+		))
 	}
 
 	/// Returns rlp at the given index, panics if no rlp at that index
@@ -75,7 +80,7 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view {
 	/// Returns decoded value at the given index, panics not present or valid at that index
 	pub fn val_at<T>(&self, index: usize) -> T where T : Decodable {
 		self.expect_valid_rlp(self.rlp.val_at(index))
-	} 
+	}
 
 	/// Returns decoded list of values, panics if rlp is invalid
 	pub fn list_at<T>(&self, index: usize) -> Vec<T> where T: Decodable {

ethcore/sync/Cargo.toml

@@ -30,6 +30,7 @@ heapsize = "0.4"
 semver = "0.9"
 smallvec = { version = "0.4", features = ["heapsizeof"] }
 parking_lot = "0.5"
+trace-time = { path = "../../util/trace-time" }
 ipnetwork = "0.12.6"
 
 [dev-dependencies]

ethcore/sync/src/api.rs

@@ -25,7 +25,7 @@ use network::{NetworkProtocolHandler, NetworkContext, HostInfo, PeerId, Protocol
 use ethereum_types::{H256, H512, U256};
 use io::{TimerToken};
 use ethcore::ethstore::ethkey::Secret;
-use ethcore::client::{BlockChainClient, ChainNotify, ChainMessageType};
+use ethcore::client::{BlockChainClient, ChainNotify, ChainRoute, ChainMessageType};
 use ethcore::snapshot::SnapshotService;
 use ethcore::header::BlockNumber;
 use sync_io::NetSyncIo;
@@ -33,7 +33,7 @@ use chain::{ChainSync, SyncStatus as EthSyncStatus};
 use std::net::{SocketAddr, AddrParseError};
 use std::str::FromStr;
 use parking_lot::RwLock;
-use chain::{ETH_PACKET_COUNT, SNAPSHOT_SYNC_PACKET_COUNT, ETH_PROTOCOL_VERSION_63, ETH_PROTOCOL_VERSION_62,
+use chain::{ETH_PROTOCOL_VERSION_63, ETH_PROTOCOL_VERSION_62,
 	PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3};
 use light::client::AsLightClient;
 use light::Provider;
@@ -202,10 +202,8 @@ pub struct AttachedProtocol {
 	pub handler: Arc<NetworkProtocolHandler + Send + Sync>,
 	/// 3-character ID for the protocol.
 	pub protocol_id: ProtocolId,
-	/// Packet count.
-	pub packet_count: u8,
-	/// Supported versions.
-	pub versions: &'static [u8],
+	/// Supported versions and their packet counts.
+	pub versions: &'static [(u8, u8)],
 }
 
 impl AttachedProtocol {
@@ -213,7 +211,6 @@ impl AttachedProtocol {
 		let res = network.register_protocol(
 			self.handler.clone(),
 			self.protocol_id,
-			self.packet_count,
 			self.versions
 		);
 
@@ -379,10 +376,12 @@ impl NetworkProtocolHandler for SyncProtocolHandler {
 	}
 
 	fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) {
+		trace_time!("sync::read");
 		ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data);
 	}
 
 	fn connected(&self, io: &NetworkContext, peer: &PeerId) {
+		trace_time!("sync::connected");
 		// If warp protocol is supported only allow warp handshake
 		let warp_protocol = io.protocol_version(WARP_SYNC_PROTOCOL_ID, *peer).unwrap_or(0) != 0;
 		let warp_context = io.subprotocol_name() == WARP_SYNC_PROTOCOL_ID;
@@ -392,12 +391,14 @@ impl NetworkProtocolHandler for SyncProtocolHandler {
 	}
 
 	fn disconnected(&self, io: &NetworkContext, peer: &PeerId) {
+		trace_time!("sync::disconnected");
 		if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID {
 			self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer);
 		}
 	}
 
 	fn timeout(&self, io: &NetworkContext, _timer: TimerToken) {
+		trace_time!("sync::timeout");
 		let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay);
 		self.sync.write().maintain_peers(&mut io);
 		self.sync.write().maintain_sync(&mut io);
@@ -409,11 +410,10 @@ impl ChainNotify for EthSync {
 	fn new_blocks(&self,
 		imported: Vec<H256>,
 		invalid: Vec<H256>,
-		enacted: Vec<H256>,
-		retracted: Vec<H256>,
+		route: ChainRoute,
 		sealed: Vec<H256>,
 		proposed: Vec<Bytes>,
-		_duration: u64)
+		_duration: Duration)
 	{
 		use light::net::Announcement;
 
@@ -424,8 +424,8 @@ impl ChainNotify for EthSync {
 				&mut sync_io,
 				&imported,
 				&invalid,
-				&enacted,
-				&retracted,
+				route.enacted(),
+				route.retracted(),
 				&sealed,
 				&proposed);
 		});
@@ -452,19 +452,19 @@ impl ChainNotify for EthSync {
 
 	fn start(&self) {
 		match self.network.start().map_err(Into::into) {
-			Err(ErrorKind::Io(ref e)) if  e.kind() == io::ErrorKind::AddrInUse => warn!("Network port {:?} is already in use, make sure that another instance of an Ethereum client is not running or change the port using the --port option.", self.network.config().listen_address.expect("Listen address is not set.")),
+			Err(ErrorKind::Io(ref e)) if e.kind() == io::ErrorKind::AddrInUse => warn!("Network port {:?} is already in use, make sure that another instance of an Ethereum client is not running or change the port using the --port option.", self.network.config().listen_address.expect("Listen address is not set.")),
 			Err(err) => warn!("Error starting network: {}", err),
 			_ => {},
 		}
-		self.network.register_protocol(self.eth_handler.clone(), self.subprotocol_name, ETH_PACKET_COUNT, &[ETH_PROTOCOL_VERSION_62, ETH_PROTOCOL_VERSION_63])
+		self.network.register_protocol(self.eth_handler.clone(), self.subprotocol_name, &[ETH_PROTOCOL_VERSION_62, ETH_PROTOCOL_VERSION_63])
 			.unwrap_or_else(|e| warn!("Error registering ethereum protocol: {:?}", e));
 		// register the warp sync subprotocol
-		self.network.register_protocol(self.eth_handler.clone(), WARP_SYNC_PROTOCOL_ID, SNAPSHOT_SYNC_PACKET_COUNT, &[PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3])
+		self.network.register_protocol(self.eth_handler.clone(), WARP_SYNC_PROTOCOL_ID, &[PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3])
 			.unwrap_or_else(|e| warn!("Error registering snapshot sync protocol: {:?}", e));
 
 		// register the light protocol.
 		if let Some(light_proto) = self.light_proto.as_ref().map(|x| x.clone()) {
-			self.network.register_protocol(light_proto, self.light_subprotocol_name, ::light::net::PACKET_COUNT, ::light::net::PROTOCOL_VERSIONS)
+			self.network.register_protocol(light_proto, self.light_subprotocol_name, ::light::net::PROTOCOL_VERSIONS)
 				.unwrap_or_else(|e| warn!("Error registering light client protocol: {:?}", e));
 		}
 
@@ -625,7 +625,7 @@ impl NetworkConfiguration {
 			config_path: self.config_path,
 			net_config_path: self.net_config_path,
 			listen_address: match self.listen_address { None => None, Some(addr) => Some(SocketAddr::from_str(&addr)?) },
-			public_address:  match self.public_address { None => None, Some(addr) => Some(SocketAddr::from_str(&addr)?) },
+			public_address: match self.public_address { None => None, Some(addr) => Some(SocketAddr::from_str(&addr)?) },
 			udp_port: self.udp_port,
 			nat_enabled: self.nat_enabled,
 			discovery_enabled: self.discovery_enabled,
@@ -824,7 +824,7 @@ impl ManageNetwork for LightSync {
 
 		let light_proto = self.proto.clone();
 
-		self.network.register_protocol(light_proto, self.subprotocol_name, ::light::net::PACKET_COUNT, ::light::net::PROTOCOL_VERSIONS)
+		self.network.register_protocol(light_proto, self.subprotocol_name, ::light::net::PROTOCOL_VERSIONS)
 			.unwrap_or_else(|e| warn!("Error registering light client protocol: {:?}", e));
 
 		for proto in &self.attached_protos { proto.register(&self.network) }

ethcore/sync/src/block_sync.rs

@@ -496,7 +496,7 @@ impl BlockDownloader {
 			}
 
 			let result = if let Some(receipts) = receipts {
-				io.chain().import_block_with_receipts(block, receipts)
+				io.chain().queue_ancient_block(block, receipts)
 			} else {
 				io.chain().import_block(block)
 			};

ethcore/sync/src/blocks.rs

@@ -266,7 +266,7 @@ impl BlockCollection {
 		}
 	}
 
-	/// Get a valid chain of blocks ordered in descending order and ready for importing into blockchain.
+	/// Get a valid chain of blocks ordered in ascending order and ready for importing into blockchain.
 	pub fn drain(&mut self) -> Vec<BlockAndReceipts> {
 		if self.blocks.is_empty() || self.head.is_none() {
 			return Vec::new();

ethcore/sync/src/chain/handler.rs

@@ -0,0 +1,853 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use api::WARP_SYNC_PROTOCOL_ID;
+use block_sync::{BlockDownloaderImportError as DownloaderImportError, DownloadAction};
+use bytes::Bytes;
+use ethcore::client::{BlockStatus, BlockId, BlockImportError, BlockImportErrorKind};
+use ethcore::error::*;
+use ethcore::header::{BlockNumber, Header as BlockHeader};
+use ethcore::snapshot::{ManifestData, RestorationStatus};
+use ethereum_types::{H256, U256};
+use hash::keccak;
+use network::PeerId;
+use rlp::Rlp;
+use snapshot::ChunkType;
+use std::cmp;
+use std::collections::HashSet;
+use std::time::Instant;
+use sync_io::SyncIo;
+
+use super::{
+	BlockSet,
+	ChainSync,
+	ForkConfirmation,
+	PacketDecodeError,
+	PeerAsking,
+	PeerInfo,
+	SyncRequester,
+	SyncState,
+	ETH_PROTOCOL_VERSION_62,
+	ETH_PROTOCOL_VERSION_63,
+	MAX_NEW_BLOCK_AGE,
+	MAX_NEW_HASHES,
+	PAR_PROTOCOL_VERSION_1,
+	PAR_PROTOCOL_VERSION_3,
+	BLOCK_BODIES_PACKET,
+	BLOCK_HEADERS_PACKET,
+	NEW_BLOCK_HASHES_PACKET,
+	NEW_BLOCK_PACKET,
+	PRIVATE_TRANSACTION_PACKET,
+	RECEIPTS_PACKET,
+	SIGNED_PRIVATE_TRANSACTION_PACKET,
+	SNAPSHOT_DATA_PACKET,
+	SNAPSHOT_MANIFEST_PACKET,
+	STATUS_PACKET,
+	TRANSACTIONS_PACKET,
+};
+
+/// The Chain Sync Handler: handles responses from peers
+pub struct SyncHandler;
+
+impl SyncHandler {
+	/// Handle incoming packet from peer
+	pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) {
+		if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) {
+			debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer));
+			return;
+		}
+		let rlp = Rlp::new(data);
+		let result = match packet_id {
+			STATUS_PACKET => SyncHandler::on_peer_status(sync, io, peer, &rlp),
+			TRANSACTIONS_PACKET => SyncHandler::on_peer_transactions(sync, io, peer, &rlp),
+			BLOCK_HEADERS_PACKET => SyncHandler::on_peer_block_headers(sync, io, peer, &rlp),
+			BLOCK_BODIES_PACKET => SyncHandler::on_peer_block_bodies(sync, io, peer, &rlp),
+			RECEIPTS_PACKET => SyncHandler::on_peer_block_receipts(sync, io, peer, &rlp),
+			NEW_BLOCK_PACKET => SyncHandler::on_peer_new_block(sync, io, peer, &rlp),
+			NEW_BLOCK_HASHES_PACKET => SyncHandler::on_peer_new_hashes(sync, io, peer, &rlp),
+			SNAPSHOT_MANIFEST_PACKET => SyncHandler::on_snapshot_manifest(sync, io, peer, &rlp),
+			SNAPSHOT_DATA_PACKET => SyncHandler::on_snapshot_data(sync, io, peer, &rlp),
+			PRIVATE_TRANSACTION_PACKET => SyncHandler::on_private_transaction(sync, io, peer, &rlp),
+			SIGNED_PRIVATE_TRANSACTION_PACKET => SyncHandler::on_signed_private_transaction(sync, io, peer, &rlp),
+			_ => {
+				debug!(target: "sync", "{}: Unknown packet {}", peer, packet_id);
+				Ok(())
+			}
+		};
+		result.unwrap_or_else(|e| {
+			debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e);
+		})
+	}
+
+	/// Called when peer sends us new consensus packet
+	pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		trace!(target: "sync", "Received consensus packet from {:?}", peer_id);
+		io.chain().queue_consensus_message(r.as_raw().to_vec());
+		Ok(())
+	}
+
+	/// Called by peer when it is disconnecting
+	pub fn on_peer_aborting(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) {
+		trace!(target: "sync", "== Disconnecting {}: {}", peer_id, io.peer_info(peer_id));
+		sync.handshaking_peers.remove(&peer_id);
+		if sync.peers.contains_key(&peer_id) {
+			debug!(target: "sync", "Disconnected {}", peer_id);
+			sync.clear_peer_download(peer_id);
+			sync.peers.remove(&peer_id);
+			sync.active_peers.remove(&peer_id);
+
+			if sync.state == SyncState::SnapshotManifest {
+				// Check if we are asking other peers for
+				// the snapshot manifest as well.
+				// If not, return to initial state
+				let still_asking_manifest = sync.peers.iter()
+					.filter(|&(id, p)| sync.active_peers.contains(id) && p.asking == PeerAsking::SnapshotManifest)
+					.next().is_none();
+
+				if still_asking_manifest {
+					sync.state = ChainSync::get_init_state(sync.warp_sync, io.chain());
+				}
+			}
+			sync.continue_sync(io);
+		}
+	}
+
+	/// Called when a new peer is connected
+	pub fn on_peer_connected(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId) {
+		trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer));
+		if let Err(e) = sync.send_status(io, peer) {
+			debug!(target:"sync", "Error sending status request: {:?}", e);
+			io.disconnect_peer(peer);
+		} else {
+			sync.handshaking_peers.insert(peer, Instant::now());
+		}
+	}
+
+	/// Called by peer once it has new block bodies
+	pub fn on_peer_new_block(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring new block from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		let difficulty: U256 = r.val_at(1)?;
+		if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+			if peer.difficulty.map_or(true, |pd| difficulty > pd) {
+				peer.difficulty = Some(difficulty);
+			}
+		}
+		let block_rlp = r.at(0)?;
+		let header_rlp = block_rlp.at(0)?;
+		let h = keccak(&header_rlp.as_raw());
+		trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h);
+		let header: BlockHeader = header_rlp.as_val()?;
+		if header.number() > sync.highest_block.unwrap_or(0) {
+			sync.highest_block = Some(header.number());
+		}
+		let mut unknown = false;
+		{
+			if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+				peer.latest_hash = header.hash();
+			}
+		}
+		let last_imported_number = sync.new_blocks.last_imported_block_number();
+		if last_imported_number > header.number() && last_imported_number - header.number() > MAX_NEW_BLOCK_AGE {
+			trace!(target: "sync", "Ignored ancient new block {:?}", h);
+			io.disable_peer(peer_id);
+			return Ok(());
+		}
+		match io.chain().import_block(block_rlp.as_raw().to_vec()) {
+			Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyInChain), _)) => {
+				trace!(target: "sync", "New block already in chain {:?}", h);
+			},
+			Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyQueued), _)) => {
+				trace!(target: "sync", "New block already queued {:?}", h);
+			},
+			Ok(_) => {
+				// abort current download of the same block
+				sync.complete_sync(io);
+				sync.new_blocks.mark_as_known(&header.hash(), header.number());
+				trace!(target: "sync", "New block queued {:?} ({})", h, header.number());
+			},
+			Err(BlockImportError(BlockImportErrorKind::Block(BlockError::UnknownParent(p)), _)) => {
+				unknown = true;
+				trace!(target: "sync", "New block with unknown parent ({:?}) {:?}", p, h);
+			},
+			Err(e) => {
+				debug!(target: "sync", "Bad new block {:?} : {:?}", h, e);
+				io.disable_peer(peer_id);
+			}
+		};
+		if unknown {
+			if sync.state != SyncState::Idle {
+				trace!(target: "sync", "NewBlock ignored while seeking");
+			} else {
+				trace!(target: "sync", "New unknown block {:?}", h);
+				//TODO: handle too many unknown blocks
+				sync.sync_peer(io, peer_id, true);
+			}
+		}
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Handles `NewHashes` packet. Initiates headers download for any unknown hashes.
+	pub fn on_peer_new_hashes(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring new hashes from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		let hashes: Vec<_> = r.iter().take(MAX_NEW_HASHES).map(|item| (item.val_at::<H256>(0), item.val_at::<BlockNumber>(1))).collect();
+		if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+			// Peer has new blocks with unknown difficulty
+			peer.difficulty = None;
+			if let Some(&(Ok(ref h), _)) = hashes.last() {
+				peer.latest_hash = h.clone();
+			}
+		}
+		if sync.state != SyncState::Idle {
+			trace!(target: "sync", "Ignoring new hashes since we're already downloading.");
+			let max = r.iter().take(MAX_NEW_HASHES).map(|item| item.val_at::<BlockNumber>(1).unwrap_or(0)).fold(0u64, cmp::max);
+			if max > sync.highest_block.unwrap_or(0) {
+				sync.highest_block = Some(max);
+			}
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		trace!(target: "sync", "{} -> NewHashes ({} entries)", peer_id, r.item_count()?);
+		let mut max_height: BlockNumber = 0;
+		let mut new_hashes = Vec::new();
+		let last_imported_number = sync.new_blocks.last_imported_block_number();
+		for (rh, rn) in hashes {
+			let hash = rh?;
+			let number = rn?;
+			if number > sync.highest_block.unwrap_or(0) {
+				sync.highest_block = Some(number);
+			}
+			if sync.new_blocks.is_downloading(&hash) {
+				continue;
+			}
+			if last_imported_number > number && last_imported_number - number > MAX_NEW_BLOCK_AGE {
+				trace!(target: "sync", "Ignored ancient new block hash {:?}", hash);
+				io.disable_peer(peer_id);
+				continue;
+			}
+			match io.chain().block_status(BlockId::Hash(hash.clone())) {
+				BlockStatus::InChain  => {
+					trace!(target: "sync", "New block hash already in chain {:?}", hash);
+				},
+				BlockStatus::Queued => {
+					trace!(target: "sync", "New hash block already queued {:?}", hash);
+				},
+				BlockStatus::Unknown | BlockStatus::Pending => {
+					new_hashes.push(hash.clone());
+					if number > max_height {
+						trace!(target: "sync", "New unknown block hash {:?}", hash);
+						if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+							peer.latest_hash = hash.clone();
+						}
+						max_height = number;
+					}
+				},
+				BlockStatus::Bad => {
+					debug!(target: "sync", "Bad new block hash {:?}", hash);
+					io.disable_peer(peer_id);
+					return Ok(());
+				}
+			}
+		};
+		if max_height != 0 {
+			trace!(target: "sync", "Downloading blocks for new hashes");
+			sync.new_blocks.reset_to(new_hashes);
+			sync.state = SyncState::NewBlocks;
+			sync.sync_peer(io, peer_id, true);
+		}
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called by peer once it has new block bodies
+	fn on_peer_block_bodies(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		sync.clear_peer_download(peer_id);
+		let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks);
+		if !sync.reset_peer_asking(peer_id, PeerAsking::BlockBodies) {
+			trace!(target: "sync", "{}: Ignored unexpected bodies", peer_id);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		let item_count = r.item_count()?;
+		trace!(target: "sync", "{} -> BlockBodies ({} entries), set = {:?}", peer_id, item_count, block_set);
+		if item_count == 0 {
+			sync.deactivate_peer(io, peer_id);
+		}
+		else if sync.state == SyncState::Waiting {
+			trace!(target: "sync", "Ignored block bodies while waiting");
+		}
+		else
+		{
+			let result = {
+				let downloader = match block_set {
+					BlockSet::NewBlocks => &mut sync.new_blocks,
+					BlockSet::OldBlocks => match sync.old_blocks {
+						None => {
+							trace!(target: "sync", "Ignored block headers while block download is inactive");
+							sync.continue_sync(io);
+							return Ok(());
+						},
+						Some(ref mut blocks) => blocks,
+					}
+				};
+				downloader.import_bodies(io, r)
+			};
+
+			match result {
+				Err(DownloaderImportError::Invalid) => {
+					io.disable_peer(peer_id);
+					sync.deactivate_peer(io, peer_id);
+					sync.continue_sync(io);
+					return Ok(());
+				},
+				Err(DownloaderImportError::Useless) => {
+					sync.deactivate_peer(io, peer_id);
+				},
+				Ok(()) => (),
+			}
+
+			sync.collect_blocks(io, block_set);
+			sync.sync_peer(io, peer_id, false);
+		}
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	fn on_peer_fork_header(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		{
+			let peer = sync.peers.get_mut(&peer_id).expect("Is only called when peer is present in peers");
+			peer.asking = PeerAsking::Nothing;
+			let item_count = r.item_count()?;
+			let (fork_number, fork_hash) = sync.fork_block.expect("ForkHeader request is sent only fork block is Some; qed").clone();
+
+			if item_count == 0 || item_count != 1 {
+				trace!(target: "sync", "{}: Chain is too short to confirm the block", peer_id);
+				peer.confirmation = ForkConfirmation::TooShort;
+
+			} else {
+				let header = r.at(0)?.as_raw();
+				if keccak(&header) != fork_hash {
+					trace!(target: "sync", "{}: Fork mismatch", peer_id);
+					io.disable_peer(peer_id);
+					return Ok(());
+				}
+
+				trace!(target: "sync", "{}: Confirmed peer", peer_id);
+				peer.confirmation = ForkConfirmation::Confirmed;
+
+				if !io.chain_overlay().read().contains_key(&fork_number) {
+					trace!(target: "sync", "Inserting (fork) block {} header", fork_number);
+					io.chain_overlay().write().insert(fork_number, header.to_vec());
+				}
+			}
+		}
+
+		sync.sync_peer(io, peer_id, false);
+		return Ok(());
+	}
+
+	/// Called by peer once it has new block headers during sync
+	fn on_peer_block_headers(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		let is_fork_header_request = match sync.peers.get(&peer_id) {
+			Some(peer) if peer.asking == PeerAsking::ForkHeader => true,
+			_ => false,
+		};
+
+		if is_fork_header_request {
+			return SyncHandler::on_peer_fork_header(sync, io, peer_id, r);
+		}
+
+		sync.clear_peer_download(peer_id);
+		let expected_hash = sync.peers.get(&peer_id).and_then(|p| p.asking_hash);
+		let allowed = sync.peers.get(&peer_id).map(|p| p.is_allowed()).unwrap_or(false);
+		let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks);
+		if !sync.reset_peer_asking(peer_id, PeerAsking::BlockHeaders) || expected_hash.is_none() || !allowed {
+			trace!(target: "sync", "{}: Ignored unexpected headers, expected_hash = {:?}", peer_id, expected_hash);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		let item_count = r.item_count()?;
+		trace!(target: "sync", "{} -> BlockHeaders ({} entries), state = {:?}, set = {:?}", peer_id, item_count, sync.state, block_set);
+		if (sync.state == SyncState::Idle || sync.state == SyncState::WaitingPeers) && sync.old_blocks.is_none() {
+			trace!(target: "sync", "Ignored unexpected block headers");
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		if sync.state == SyncState::Waiting {
+			trace!(target: "sync", "Ignored block headers while waiting");
+			sync.continue_sync(io);
+			return Ok(());
+		}
+
+		let result =  {
+			let downloader = match block_set {
+				BlockSet::NewBlocks => &mut sync.new_blocks,
+				BlockSet::OldBlocks => {
+					match sync.old_blocks {
+						None => {
+							trace!(target: "sync", "Ignored block headers while block download is inactive");
+							sync.continue_sync(io);
+							return Ok(());
+						},
+						Some(ref mut blocks) => blocks,
+					}
+				}
+			};
+			downloader.import_headers(io, r, expected_hash)
+		};
+
+		match result {
+			Err(DownloaderImportError::Useless) => {
+				sync.deactivate_peer(io, peer_id);
+			},
+			Err(DownloaderImportError::Invalid) => {
+				io.disable_peer(peer_id);
+				sync.deactivate_peer(io, peer_id);
+				sync.continue_sync(io);
+				return Ok(());
+			},
+			Ok(DownloadAction::Reset) => {
+				// mark all outstanding requests as expired
+				trace!("Resetting downloads for {:?}", block_set);
+				for (_, ref mut p) in sync.peers.iter_mut().filter(|&(_, ref p)| p.block_set == Some(block_set)) {
+					p.reset_asking();
+				}
+
+			}
+			Ok(DownloadAction::None) => {},
+		}
+
+		sync.collect_blocks(io, block_set);
+		// give a task to the same peer first if received valuable headers.
+		sync.sync_peer(io, peer_id, false);
+		// give tasks to other peers
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called by peer once it has new block receipts
+	fn on_peer_block_receipts(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		sync.clear_peer_download(peer_id);
+		let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks);
+		if !sync.reset_peer_asking(peer_id, PeerAsking::BlockReceipts) {
+			trace!(target: "sync", "{}: Ignored unexpected receipts", peer_id);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		let item_count = r.item_count()?;
+		trace!(target: "sync", "{} -> BlockReceipts ({} entries)", peer_id, item_count);
+		if item_count == 0 {
+			sync.deactivate_peer(io, peer_id);
+		}
+		else if sync.state == SyncState::Waiting {
+			trace!(target: "sync", "Ignored block receipts while waiting");
+		}
+		else
+		{
+			let result = {
+				let downloader = match block_set {
+					BlockSet::NewBlocks => &mut sync.new_blocks,
+					BlockSet::OldBlocks => match sync.old_blocks {
+						None => {
+							trace!(target: "sync", "Ignored block headers while block download is inactive");
+							sync.continue_sync(io);
+							return Ok(());
+						},
+						Some(ref mut blocks) => blocks,
+					}
+				};
+				downloader.import_receipts(io, r)
+			};
+
+			match result {
+				Err(DownloaderImportError::Invalid) => {
+					io.disable_peer(peer_id);
+					sync.deactivate_peer(io, peer_id);
+					sync.continue_sync(io);
+					return Ok(());
+				},
+				Err(DownloaderImportError::Useless) => {
+					sync.deactivate_peer(io, peer_id);
+				},
+				Ok(()) => (),
+			}
+
+			sync.collect_blocks(io, block_set);
+			sync.sync_peer(io, peer_id, false);
+		}
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called when snapshot manifest is downloaded from a peer.
+	fn on_snapshot_manifest(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		sync.clear_peer_download(peer_id);
+		if !sync.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || sync.state != SyncState::SnapshotManifest {
+			trace!(target: "sync", "{}: Ignored unexpected/expired manifest", peer_id);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+
+		let manifest_rlp = r.at(0)?;
+		let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) {
+			Err(e) => {
+				trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e);
+				io.disable_peer(peer_id);
+				sync.continue_sync(io);
+				return Ok(());
+			}
+			Ok(manifest) => manifest,
+		};
+
+		let is_supported_version = io.snapshot_service().supported_versions()
+			.map_or(false, |(l, h)| manifest.version >= l && manifest.version <= h);
+
+		if !is_supported_version {
+			trace!(target: "sync", "{}: Snapshot manifest version not supported: {}", peer_id, manifest.version);
+			io.disable_peer(peer_id);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+		sync.snapshot.reset_to(&manifest, &keccak(manifest_rlp.as_raw()));
+		io.snapshot_service().begin_restore(manifest);
+		sync.state = SyncState::SnapshotData;
+
+		// give a task to the same peer first.
+		sync.sync_peer(io, peer_id, false);
+		// give tasks to other peers
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called when snapshot data is downloaded from a peer.
+	fn on_snapshot_data(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		sync.clear_peer_download(peer_id);
+		if !sync.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || (sync.state != SyncState::SnapshotData && sync.state != SyncState::SnapshotWaiting) {
+			trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id);
+			sync.continue_sync(io);
+			return Ok(());
+		}
+
+		// check service status
+		let status = io.snapshot_service().status();
+		match status {
+			RestorationStatus::Inactive | RestorationStatus::Failed => {
+				trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id);
+				sync.state = SyncState::WaitingPeers;
+
+				// only note bad if restoration failed.
+				if let (Some(hash), RestorationStatus::Failed) = (sync.snapshot.snapshot_hash(), status) {
+					trace!(target: "sync", "Noting snapshot hash {} as bad", hash);
+					sync.snapshot.note_bad(hash);
+				}
+
+				sync.snapshot.clear();
+				sync.continue_sync(io);
+				return Ok(());
+			},
+			RestorationStatus::Initializing  { .. } => {
+				trace!(target: "warp", "{}: Snapshot restoration is initializing", peer_id);
+				return Ok(());
+			}
+			RestorationStatus::Ongoing { .. } => {
+				trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id);
+			},
+		}
+
+		let snapshot_data: Bytes = r.val_at(0)?;
+		match sync.snapshot.validate_chunk(&snapshot_data) {
+			Ok(ChunkType::Block(hash)) => {
+				trace!(target: "sync", "{}: Processing block chunk", peer_id);
+				io.snapshot_service().restore_block_chunk(hash, snapshot_data);
+			}
+			Ok(ChunkType::State(hash)) => {
+				trace!(target: "sync", "{}: Processing state chunk", peer_id);
+				io.snapshot_service().restore_state_chunk(hash, snapshot_data);
+			}
+			Err(()) => {
+				trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id);
+				io.disconnect_peer(peer_id);
+				sync.continue_sync(io);
+				return Ok(());
+			}
+		}
+
+		if sync.snapshot.is_complete() {
+			// wait for snapshot restoration process to complete
+			sync.state = SyncState::SnapshotWaiting;
+		}
+		// give a task to the same peer first.
+		sync.sync_peer(io, peer_id, false);
+		// give tasks to other peers
+		sync.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called by peer to report status
+	fn on_peer_status(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		sync.handshaking_peers.remove(&peer_id);
+		let protocol_version: u8 = r.val_at(0)?;
+		let warp_protocol = io.protocol_version(&WARP_SYNC_PROTOCOL_ID, peer_id) != 0;
+		let peer = PeerInfo {
+			protocol_version: protocol_version,
+			network_id: r.val_at(1)?,
+			difficulty: Some(r.val_at(2)?),
+			latest_hash: r.val_at(3)?,
+			genesis: r.val_at(4)?,
+			asking: PeerAsking::Nothing,
+			asking_blocks: Vec::new(),
+			asking_hash: None,
+			ask_time: Instant::now(),
+			last_sent_transactions: HashSet::new(),
+			expired: false,
+			confirmation: if sync.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed },
+			asking_snapshot_data: None,
+			snapshot_hash: if warp_protocol { Some(r.val_at(5)?) } else { None },
+			snapshot_number: if warp_protocol { Some(r.val_at(6)?) } else { None },
+			block_set: None,
+		};
+
+		trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{}, snapshot:{:?})",
+			peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis, peer.snapshot_number);
+		if io.is_expired() {
+			trace!(target: "sync", "Status packet from expired session {}:{}", peer_id, io.peer_info(peer_id));
+			return Ok(());
+		}
+
+		if sync.peers.contains_key(&peer_id) {
+			debug!(target: "sync", "Unexpected status packet from {}:{}", peer_id, io.peer_info(peer_id));
+			return Ok(());
+		}
+		let chain_info = io.chain().chain_info();
+		if peer.genesis != chain_info.genesis_hash {
+			io.disable_peer(peer_id);
+			trace!(target: "sync", "Peer {} genesis hash mismatch (ours: {}, theirs: {})", peer_id, chain_info.genesis_hash, peer.genesis);
+			return Ok(());
+		}
+		if peer.network_id != sync.network_id {
+			io.disable_peer(peer_id);
+			trace!(target: "sync", "Peer {} network id mismatch (ours: {}, theirs: {})", peer_id, sync.network_id, peer.network_id);
+			return Ok(());
+		}
+
+		if false
+			|| (warp_protocol && (peer.protocol_version < PAR_PROTOCOL_VERSION_1.0 || peer.protocol_version > PAR_PROTOCOL_VERSION_3.0))
+			|| (!warp_protocol && (peer.protocol_version < ETH_PROTOCOL_VERSION_62.0 || peer.protocol_version > ETH_PROTOCOL_VERSION_63.0))
+		{
+			io.disable_peer(peer_id);
+			trace!(target: "sync", "Peer {} unsupported eth protocol ({})", peer_id, peer.protocol_version);
+			return Ok(());
+		}
+
+		if sync.sync_start_time.is_none() {
+			sync.sync_start_time = Some(Instant::now());
+		}
+
+		sync.peers.insert(peer_id.clone(), peer);
+		// Don't activate peer immediatelly when searching for common block.
+		// Let the current sync round complete first.
+		sync.active_peers.insert(peer_id.clone());
+		debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id));
+
+		match sync.fork_block {
+			Some((fork_block, _)) => {
+				SyncRequester::request_fork_header(sync, io, peer_id, fork_block);
+			},
+			_ => {
+				// when there's no `fork_block` defined we initialize the peer with
+				// `confirmation: ForkConfirmation::Confirmed`.
+				sync.sync_peer(io, peer_id, false);
+			}
+		}
+
+		Ok(())
+	}
+
+	/// Called when peer sends us new transactions
+	fn on_peer_transactions(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		// Accept transactions only when fully synced
+		if !io.is_chain_queue_empty() || (sync.state != SyncState::Idle && sync.state != SyncState::NewBlocks) {
+			trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id);
+			return Ok(());
+		}
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "{} Ignoring transactions from unconfirmed/unknown peer", peer_id);
+			return Ok(());
+		}
+
+		let item_count = r.item_count()?;
+		trace!(target: "sync", "{:02} -> Transactions ({} entries)", peer_id, item_count);
+		let mut transactions = Vec::with_capacity(item_count);
+		for i in 0 .. item_count {
+			let rlp = r.at(i)?;
+			let tx = rlp.as_raw().to_vec();
+			transactions.push(tx);
+		}
+		io.chain().queue_transactions(transactions, peer_id);
+		Ok(())
+	}
+
+	/// Called when peer sends us signed private transaction packet
+	fn on_signed_private_transaction(sync: &ChainSync, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id);
+			return Ok(());
+		}
+
+		trace!(target: "sync", "Received signed private transaction packet from {:?}", peer_id);
+		if let Err(e) = sync.private_tx_handler.import_signed_private_transaction(r.as_raw()) {
+			trace!(target: "sync", "Ignoring the message, error queueing: {}", e);
+		}
+		Ok(())
+	}
+
+	/// Called when peer sends us new private transaction packet
+	fn on_private_transaction(sync: &ChainSync, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> {
+		if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id);
+			return Ok(());
+		}
+
+		trace!(target: "sync", "Received private transaction packet from {:?}", peer_id);
+
+		if let Err(e) = sync.private_tx_handler.import_private_transaction(r.as_raw()) {
+			trace!(target: "sync", "Ignoring the message, error queueing: {}", e);
+		}
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use ethcore::client::{ChainInfo, EachBlockWith, TestBlockChainClient};
+	use parking_lot::RwLock;
+	use rlp::{Rlp};
+	use std::collections::{VecDeque};
+	use tests::helpers::{TestIo};
+	use tests::snapshot::TestSnapshotService;
+
+	use super::*;
+	use super::super::tests::{
+		dummy_sync_with_peer,
+		get_dummy_block,
+		get_dummy_blocks,
+		get_dummy_hashes,
+	};
+
+	#[test]
+	fn handles_peer_new_hashes() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(10, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let hashes_data = get_dummy_hashes();
+		let hashes_rlp = Rlp::new(&hashes_data);
+
+		let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp);
+
+		assert!(result.is_ok());
+	}
+
+	#[test]
+	fn handles_peer_new_block_malformed() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(10, EachBlockWith::Uncle);
+
+		let block_data = get_dummy_block(11, client.chain_info().best_block_hash);
+
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		//sync.have_common_block = true;
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let block = Rlp::new(&block_data);
+
+		let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block);
+
+		assert!(result.is_err());
+	}
+
+	#[test]
+	fn handles_peer_new_block() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(10, EachBlockWith::Uncle);
+
+		let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash);
+
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let block = Rlp::new(&block_data);
+
+		let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block);
+
+		assert!(result.is_ok());
+	}
+
+	#[test]
+	fn handles_peer_new_block_empty() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(10, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let empty_data = vec![];
+		let block = Rlp::new(&empty_data);
+
+		let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block);
+
+		assert!(result.is_err());
+	}
+
+	#[test]
+	fn handles_peer_new_hashes_empty() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(10, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let empty_hashes_data = vec![];
+		let hashes_rlp = Rlp::new(&empty_hashes_data);
+
+		let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp);
+
+		assert!(result.is_ok());
+	}
+}

ethcore/sync/src/chain/propagator.rs

@@ -0,0 +1,636 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use bytes::Bytes;
+use ethereum_types::H256;
+use ethcore::client::BlockChainInfo;
+use ethcore::header::BlockNumber;
+use network::{PeerId, PacketId};
+use rand::Rng;
+use rlp::{Encodable, RlpStream};
+use sync_io::SyncIo;
+use std::cmp;
+use std::collections::HashSet;
+use transaction::SignedTransaction;
+
+use super::{
+	random,
+	ChainSync,
+	MAX_PEER_LAG_PROPAGATION,
+	MAX_PEERS_PROPAGATION,
+	MAX_TRANSACTION_PACKET_SIZE,
+	MAX_TRANSACTIONS_TO_PROPAGATE,
+	MIN_PEERS_PROPAGATION,
+	CONSENSUS_DATA_PACKET,
+	NEW_BLOCK_HASHES_PACKET,
+	NEW_BLOCK_PACKET,
+	PRIVATE_TRANSACTION_PACKET,
+	SIGNED_PRIVATE_TRANSACTION_PACKET,
+	TRANSACTIONS_PACKET,
+};
+
+/// Checks if peer is able to process service transactions
+fn accepts_service_transaction(client_id: &str) -> bool {
+	// Parity versions starting from this will accept service-transactions
+	const SERVICE_TRANSACTIONS_VERSION: (u32, u32) = (1u32, 6u32);
+	// Parity client string prefix
+	const PARITY_CLIENT_ID_PREFIX: &'static str = "Parity/v";
+
+	if !client_id.starts_with(PARITY_CLIENT_ID_PREFIX) {
+		return false;
+	}
+	let ver: Vec<u32> = client_id[PARITY_CLIENT_ID_PREFIX.len()..].split('.')
+		.take(2)
+		.filter_map(|s| s.parse().ok())
+		.collect();
+	ver.len() == 2 && (ver[0] > SERVICE_TRANSACTIONS_VERSION.0 || (ver[0] == SERVICE_TRANSACTIONS_VERSION.0 && ver[1] >= SERVICE_TRANSACTIONS_VERSION.1))
+}
+
+/// The Chain Sync Propagator: propagates data to peers
+pub struct SyncPropagator;
+
+impl SyncPropagator {
+	/// propagates latest block to a set of peers
+	pub fn propagate_blocks(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize {
+		trace!(target: "sync", "Sending NewBlocks to {:?}", peers);
+		let mut sent = 0;
+		for peer_id in peers {
+			if blocks.is_empty() {
+				let rlp =  ChainSync::create_latest_block_rlp(io.chain());
+				SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp);
+			} else {
+				for h in blocks {
+					let rlp =  ChainSync::create_new_block_rlp(io.chain(), h);
+					SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp);
+				}
+			}
+			if let Some(ref mut peer) = sync.peers.get_mut(peer_id) {
+				peer.latest_hash = chain_info.best_block_hash.clone();
+			}
+			sent += 1;
+		}
+		sent
+	}
+
+	/// propagates new known hashes to all peers
+	pub fn propagate_new_hashes(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize {
+		trace!(target: "sync", "Sending NewHashes to {:?}", peers);
+		let mut sent = 0;
+		let last_parent = *io.chain().best_block_header().parent_hash();
+		for peer_id in peers {
+			sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) {
+				Some(rlp) => {
+					{
+						if let Some(ref mut peer) = sync.peers.get_mut(peer_id) {
+							peer.latest_hash = chain_info.best_block_hash.clone();
+						}
+					}
+					SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp);
+					1
+				},
+				None => 0
+			}
+		}
+		sent
+	}
+
+	/// propagates new transactions to all peers
+	pub fn propagate_new_transactions(sync: &mut ChainSync, io: &mut SyncIo) -> usize {
+		// Early out if nobody to send to.
+		if sync.peers.is_empty() {
+			return 0;
+		}
+
+		let transactions = io.chain().ready_transactions();
+		if transactions.is_empty() {
+			return 0;
+		}
+
+		let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter()
+			.map(|tx| tx.signed())
+			.partition(|tx| !tx.gas_price.is_zero());
+
+		// usual transactions could be propagated to all peers
+		let mut affected_peers = HashSet::new();
+		if !transactions.is_empty() {
+			let peers = SyncPropagator::select_peers_for_transactions(sync, |_| true);
+			affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, peers, transactions);
+		}
+
+		// most of times service_transactions will be empty
+		// => there's no need to merge packets
+		if !service_transactions.is_empty() {
+			let service_transactions_peers = SyncPropagator::select_peers_for_transactions(sync, |peer_id| accepts_service_transaction(&io.peer_info(*peer_id)));
+			let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, service_transactions_peers, service_transactions);
+			affected_peers.extend(&service_transactions_affected_peers);
+		}
+
+		affected_peers.len()
+	}
+
+	fn propagate_transactions_to_peers(sync: &mut ChainSync, io: &mut SyncIo, peers: Vec<PeerId>, transactions: Vec<&SignedTransaction>) -> HashSet<PeerId> {
+		let all_transactions_hashes = transactions.iter()
+			.map(|tx| tx.hash())
+			.collect::<HashSet<H256>>();
+		let all_transactions_rlp = {
+			let mut packet = RlpStream::new_list(transactions.len());
+			for tx in &transactions { packet.append(&**tx); }
+			packet.out()
+		};
+
+		// Clear old transactions from stats
+		sync.transactions_stats.retain(&all_transactions_hashes);
+
+		// sqrt(x)/x scaled to max u32
+		let block_number = io.chain().chain_info().best_block_number;
+
+		let lucky_peers = {
+			peers.into_iter()
+				.filter_map(|peer_id| {
+					let stats = &mut sync.transactions_stats;
+					let peer_info = sync.peers.get_mut(&peer_id)
+						.expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed");
+
+					// Send all transactions
+					if peer_info.last_sent_transactions.is_empty() {
+						// update stats
+						for hash in &all_transactions_hashes {
+							let id = io.peer_session_info(peer_id).and_then(|info| info.id);
+							stats.propagated(hash, id, block_number);
+						}
+						peer_info.last_sent_transactions = all_transactions_hashes.clone();
+						return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone()));
+					}
+
+					// Get hashes of all transactions to send to this peer
+					let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions)
+						.take(MAX_TRANSACTIONS_TO_PROPAGATE)
+						.cloned()
+						.collect::<HashSet<_>>();
+					if to_send.is_empty() {
+						return None;
+					}
+
+					// Construct RLP
+					let (packet, to_send) = {
+						let mut to_send = to_send;
+						let mut packet = RlpStream::new();
+						packet.begin_unbounded_list();
+						let mut pushed = 0;
+						for tx in &transactions {
+							let hash = tx.hash();
+							if to_send.contains(&hash) {
+								let mut transaction = RlpStream::new();
+								tx.rlp_append(&mut transaction);
+								let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE);
+								if !appended {
+									// Maximal packet size reached just proceed with sending
+									debug!("Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len());
+									to_send = to_send.into_iter().take(pushed).collect();
+									break;
+								}
+								pushed += 1;
+							}
+						}
+						packet.complete_unbounded_list();
+						(packet, to_send)
+					};
+
+					// Update stats
+					let id = io.peer_session_info(peer_id).and_then(|info| info.id);
+					for hash in &to_send {
+						// update stats
+						stats.propagated(hash, id, block_number);
+					}
+
+					peer_info.last_sent_transactions = all_transactions_hashes
+						.intersection(&peer_info.last_sent_transactions)
+						.chain(&to_send)
+						.cloned()
+						.collect();
+					Some((peer_id, to_send.len(), packet.out()))
+				})
+				.collect::<Vec<_>>()
+		};
+
+		// Send RLPs
+		let mut peers = HashSet::new();
+		if lucky_peers.len() > 0 {
+			let mut max_sent = 0;
+			let lucky_peers_len = lucky_peers.len();
+			for (peer_id, sent, rlp) in lucky_peers {
+				peers.insert(peer_id);
+				SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp);
+				trace!(target: "sync", "{:02} <- Transactions ({} entries)", peer_id, sent);
+				max_sent = cmp::max(max_sent, sent);
+			}
+			debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len);
+		}
+
+		peers
+	}
+
+	pub fn propagate_latest_blocks(sync: &mut ChainSync, io: &mut SyncIo, sealed: &[H256]) {
+		let chain_info = io.chain().chain_info();
+		if (((chain_info.best_block_number as i64) - (sync.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION {
+			let mut peers = sync.get_lagging_peers(&chain_info);
+			if sealed.is_empty() {
+				let hashes = SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers);
+				peers = ChainSync::select_random_peers(&peers);
+				let blocks = SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers);
+				if blocks != 0 || hashes != 0 {
+					trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes);
+				}
+			} else {
+				SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers);
+				SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers);
+				trace!(target: "sync", "Sent sealed block to all peers");
+			};
+		}
+		sync.last_sent_block_number = chain_info.best_block_number;
+	}
+
+	/// Distribute valid proposed blocks to subset of current peers.
+	pub fn propagate_proposed_blocks(sync: &mut ChainSync, io: &mut SyncIo, proposed: &[Bytes]) {
+		let peers = sync.get_consensus_peers();
+		trace!(target: "sync", "Sending proposed blocks to {:?}", peers);
+		for block in proposed {
+			let rlp = ChainSync::create_block_rlp(
+				block,
+				io.chain().chain_info().total_difficulty
+			);
+			for peer_id in &peers {
+				SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone());
+			}
+		}
+	}
+
+	/// Broadcast consensus message to peers.
+	pub fn propagate_consensus_packet(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) {
+		let lucky_peers = ChainSync::select_random_peers(&sync.get_consensus_peers());
+		trace!(target: "sync", "Sending consensus packet to {:?}", lucky_peers);
+		for peer_id in lucky_peers {
+			SyncPropagator::send_packet(io, peer_id, CONSENSUS_DATA_PACKET, packet.clone());
+		}
+	}
+
+	/// Broadcast private transaction message to peers.
+	pub fn propagate_private_transaction(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) {
+		let lucky_peers = ChainSync::select_random_peers(&sync.get_private_transaction_peers());
+		trace!(target: "sync", "Sending private transaction packet to {:?}", lucky_peers);
+		for peer_id in lucky_peers {
+			SyncPropagator::send_packet(io, peer_id, PRIVATE_TRANSACTION_PACKET, packet.clone());
+		}
+	}
+
+	/// Broadcast signed private transaction message to peers.
+	pub fn propagate_signed_private_transaction(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) {
+		let lucky_peers = ChainSync::select_random_peers(&sync.get_private_transaction_peers());
+		trace!(target: "sync", "Sending signed private transaction packet to {:?}", lucky_peers);
+		for peer_id in lucky_peers {
+			SyncPropagator::send_packet(io, peer_id, SIGNED_PRIVATE_TRANSACTION_PACKET, packet.clone());
+		}
+	}
+
+	fn select_peers_for_transactions<F>(sync: &ChainSync, filter: F) -> Vec<PeerId>
+		where F: Fn(&PeerId) -> bool {
+		// sqrt(x)/x scaled to max u32
+		let fraction = ((sync.peers.len() as f64).powf(-0.5) * (u32::max_value() as f64).round()) as u32;
+		let small = sync.peers.len() < MIN_PEERS_PROPAGATION;
+
+		let mut random = random::new();
+		sync.peers.keys()
+			.cloned()
+			.filter(filter)
+			.filter(|_| small || random.next_u32() < fraction)
+			.take(MAX_PEERS_PROPAGATION)
+			.collect()
+	}
+
+	/// Generic packet sender
+	fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) {
+		if let Err(e) = sync.send(peer_id, packet_id, packet) {
+			debug!(target:"sync", "Error sending packet: {:?}", e);
+			sync.disconnect_peer(peer_id);
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use ethcore::client::{BlockInfo, ChainInfo, EachBlockWith, TestBlockChainClient};
+	use parking_lot::RwLock;
+	use private_tx::NoopPrivateTxHandler;
+	use rlp::{Rlp};
+	use std::collections::{VecDeque};
+	use tests::helpers::{TestIo};
+	use tests::snapshot::TestSnapshotService;
+
+	use super::{*, super::{*, tests::*}};
+
+	#[test]
+	fn sends_new_hashes_to_lagging_peer() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let chain_info = client.chain_info();
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let peers = sync.get_lagging_peers(&chain_info);
+		let peer_count = SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers);
+
+		// 1 message should be send
+		assert_eq!(1, io.packets.len());
+		// 1 peer should be updated
+		assert_eq!(1, peer_count);
+		// NEW_BLOCK_HASHES_PACKET
+		assert_eq!(0x01, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn sends_latest_block_to_lagging_peer() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let chain_info = client.chain_info();
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		let peers = sync.get_lagging_peers(&chain_info);
+		let peer_count = SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers);
+
+		// 1 message should be send
+		assert_eq!(1, io.packets.len());
+		// 1 peer should be updated
+		assert_eq!(1, peer_count);
+		// NEW_BLOCK_PACKET
+		assert_eq!(0x07, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn sends_sealed_block() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let hash = client.block_hash(BlockId::Number(99)).unwrap();
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
+		let chain_info = client.chain_info();
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		let peers = sync.get_lagging_peers(&chain_info);
+		let peer_count = SyncPropagator::propagate_blocks(&mut sync ,&chain_info, &mut io, &[hash.clone()], &peers);
+
+		// 1 message should be send
+		assert_eq!(1, io.packets.len());
+		// 1 peer should be updated
+		assert_eq!(1, peer_count);
+		// NEW_BLOCK_PACKET
+		assert_eq!(0x07, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn sends_proposed_block() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(2, EachBlockWith::Uncle);
+		let queue = RwLock::new(VecDeque::new());
+		let block = client.block(BlockId::Latest).unwrap().into_inner();
+		let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler));
+		sync.peers.insert(0,
+			PeerInfo {
+				// Messaging protocol
+				protocol_version: 2,
+				genesis: H256::zero(),
+				network_id: 0,
+				latest_hash: client.block_hash_delta_minus(1),
+				difficulty: None,
+				asking: PeerAsking::Nothing,
+				asking_blocks: Vec::new(),
+				asking_hash: None,
+				ask_time: Instant::now(),
+				last_sent_transactions: HashSet::new(),
+				expired: false,
+				confirmation: ForkConfirmation::Confirmed,
+				snapshot_number: None,
+				snapshot_hash: None,
+				asking_snapshot_data: None,
+				block_set: None,
+			});
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		SyncPropagator::propagate_proposed_blocks(&mut sync, &mut io, &[block]);
+
+		// 1 message should be sent
+		assert_eq!(1, io.packets.len());
+		// NEW_BLOCK_PACKET
+		assert_eq!(0x07, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn propagates_transactions() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		client.insert_transaction_to_queue();
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+		// Try to propagate same transactions for the second time
+		let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+		// Even after new block transactions should not be propagated twice
+		sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]);
+		// Try to propagate same transactions for the third time
+		let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+
+		// 1 message should be send
+		assert_eq!(1, io.packets.len());
+		// 1 peer should be updated but only once
+		assert_eq!(1, peer_count);
+		assert_eq!(0, peer_count2);
+		assert_eq!(0, peer_count3);
+		// TRANSACTIONS_PACKET
+		assert_eq!(0x02, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn does_not_propagate_new_transactions_after_new_block() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		client.insert_transaction_to_queue();
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+		io.chain.insert_transaction_to_queue();
+		// New block import should not trigger propagation.
+		// (we only propagate on timeout)
+		sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]);
+
+		// 2 message should be send
+		assert_eq!(1, io.packets.len());
+		// 1 peer should receive the message
+		assert_eq!(1, peer_count);
+		// TRANSACTIONS_PACKET
+		assert_eq!(0x02, io.packets[0].packet_id);
+	}
+
+	#[test]
+	fn does_not_fail_for_no_peers() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		client.insert_transaction_to_queue();
+		// Sync with no peers
+		let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler));
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+		sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]);
+		// Try to propagate same transactions for the second time
+		let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+
+		assert_eq!(0, io.packets.len());
+		assert_eq!(0, peer_count);
+		assert_eq!(0, peer_count2);
+	}
+
+	#[test]
+	fn propagates_transactions_without_alternating() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		client.insert_transaction_to_queue();
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		// should sent some
+		{
+			let mut io = TestIo::new(&mut client, &ss, &queue, None);
+			let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+			assert_eq!(1, io.packets.len());
+			assert_eq!(1, peer_count);
+		}
+		// Insert some more
+		client.insert_transaction_to_queue();
+		let (peer_count2, peer_count3) = {
+			let mut io = TestIo::new(&mut client, &ss, &queue, None);
+			// Propagate new transactions
+			let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+			// And now the peer should have all transactions
+			let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+			(peer_count2, peer_count3)
+		};
+
+		// 2 message should be send (in total)
+		assert_eq!(2, queue.read().len());
+		// 1 peer should be updated but only once after inserting new transaction
+		assert_eq!(1, peer_count2);
+		assert_eq!(0, peer_count3);
+		// TRANSACTIONS_PACKET
+		assert_eq!(0x02, queue.read()[0].packet_id);
+		assert_eq!(0x02, queue.read()[1].packet_id);
+	}
+
+	#[test]
+	fn should_maintain_transations_propagation_stats() {
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Uncle);
+		client.insert_transaction_to_queue();
+		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+		SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+
+		let stats = sync.transactions_stats();
+		assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.")
+	}
+
+	#[test]
+	fn should_propagate_service_transaction_to_selected_peers_only() {
+		let mut client = TestBlockChainClient::new();
+		client.insert_transaction_with_gas_price_to_queue(U256::zero());
+		let block_hash = client.block_hash_delta_minus(1);
+		let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler));
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		// when peer#1 is Geth
+		insert_dummy_peer(&mut sync, 1, block_hash);
+		io.peers_info.insert(1, "Geth".to_owned());
+		// and peer#2 is Parity, accepting service transactions
+		insert_dummy_peer(&mut sync, 2, block_hash);
+		io.peers_info.insert(2, "Parity/v1.6".to_owned());
+		// and peer#3 is Parity, discarding service transactions
+		insert_dummy_peer(&mut sync, 3, block_hash);
+		io.peers_info.insert(3, "Parity/v1.5".to_owned());
+		// and peer#4 is Parity, accepting service transactions
+		insert_dummy_peer(&mut sync, 4, block_hash);
+		io.peers_info.insert(4, "Parity/v1.7.3-ABCDEFGH".to_owned());
+
+		// and new service transaction is propagated to peers
+		SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+
+		// peer#2 && peer#4 are receiving service transaction
+		assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET
+		assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 4)); // TRANSACTIONS_PACKET
+		assert_eq!(io.packets.len(), 2);
+	}
+
+	#[test]
+	fn should_propagate_service_transaction_is_sent_as_separate_message() {
+		let mut client = TestBlockChainClient::new();
+		let tx1_hash = client.insert_transaction_to_queue();
+		let tx2_hash = client.insert_transaction_with_gas_price_to_queue(U256::zero());
+		let block_hash = client.block_hash_delta_minus(1);
+		let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler));
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		// when peer#1 is Parity, accepting service transactions
+		insert_dummy_peer(&mut sync, 1, block_hash);
+		io.peers_info.insert(1, "Parity/v1.6".to_owned());
+
+		// and service + non-service transactions are propagated to peers
+		SyncPropagator::propagate_new_transactions(&mut sync, &mut io);
+
+		// two separate packets for peer are queued:
+		// 1) with non-service-transaction
+		// 2) with service transaction
+		let sent_transactions: Vec<UnverifiedTransaction> = io.packets.iter()
+			.filter_map(|p| {
+				if p.packet_id != 0x02 || p.recipient != 1 { // TRANSACTIONS_PACKET
+					return None;
+				}
+
+				let rlp = Rlp::new(&*p.data);
+				let item_count = rlp.item_count().unwrap_or(0);
+				if item_count != 1 {
+					return None;
+				}
+
+				rlp.at(0).ok().and_then(|r| r.as_val().ok())
+			})
+			.collect();
+		assert_eq!(sent_transactions.len(), 2);
+		assert!(sent_transactions.iter().any(|tx| tx.hash() == tx1_hash));
+		assert!(sent_transactions.iter().any(|tx| tx.hash() == tx2_hash));
+	}
+}

ethcore/sync/src/chain/requester.rs

@@ -0,0 +1,155 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use api::WARP_SYNC_PROTOCOL_ID;
+use block_sync::BlockRequest;
+use bytes::Bytes;
+use ethcore::header::BlockNumber;
+use ethereum_types::H256;
+use network::{PeerId, PacketId};
+use rlp::RlpStream;
+use std::time::Instant;
+use sync_io::SyncIo;
+
+use super::{
+	BlockSet,
+	ChainSync,
+	PeerAsking,
+	ETH_PROTOCOL_VERSION_63,
+	GET_BLOCK_BODIES_PACKET,
+	GET_BLOCK_HEADERS_PACKET,
+	GET_RECEIPTS_PACKET,
+	GET_SNAPSHOT_DATA_PACKET,
+	GET_SNAPSHOT_MANIFEST_PACKET,
+};
+
+/// The Chain Sync Requester: requesting data to other peers
+pub struct SyncRequester;
+
+impl SyncRequester {
+	/// Perform block download request`
+	pub fn request_blocks(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) {
+		match request {
+			BlockRequest::Headers { start, count, skip } => {
+				SyncRequester::request_headers_by_hash(sync, io, peer_id, &start, count, skip, false, block_set);
+			},
+			BlockRequest::Bodies { hashes } => {
+				SyncRequester::request_bodies(sync, io, peer_id, hashes, block_set);
+			},
+			BlockRequest::Receipts { hashes } => {
+				SyncRequester::request_receipts(sync, io, peer_id, hashes, block_set);
+			},
+		}
+	}
+
+	/// Request block bodies from a peer
+	fn request_bodies(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, hashes: Vec<H256>, set: BlockSet) {
+		let mut rlp = RlpStream::new_list(hashes.len());
+		trace!(target: "sync", "{} <- GetBlockBodies: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set);
+		for h in &hashes {
+			rlp.append(&h.clone());
+		}
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out());
+		let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed");
+		peer.asking_blocks = hashes;
+		peer.block_set = Some(set);
+	}
+
+	/// Request headers from a peer by block number
+	pub fn request_fork_header(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, n: BlockNumber) {
+		trace!(target: "sync", "{} <- GetForkHeader: at {}", peer_id, n);
+		let mut rlp = RlpStream::new_list(4);
+		rlp.append(&n);
+		rlp.append(&1u32);
+		rlp.append(&0u32);
+		rlp.append(&0u32);
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::ForkHeader, GET_BLOCK_HEADERS_PACKET, rlp.out());
+	}
+
+	/// Find some headers or blocks to download for a peer.
+	pub fn request_snapshot_data(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) {
+		// find chunk data to download
+		if let Some(hash) = sync.snapshot.needed_chunk() {
+			if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+				peer.asking_snapshot_data = Some(hash.clone());
+			}
+			SyncRequester::request_snapshot_chunk(sync, io, peer_id, &hash);
+		}
+	}
+
+	/// Request snapshot manifest from a peer.
+	pub fn request_snapshot_manifest(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) {
+		trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id);
+		let rlp = RlpStream::new_list(0);
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out());
+	}
+
+	/// Request headers from a peer by block hash
+	fn request_headers_by_hash(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, h: &H256, count: u64, skip: u64, reverse: bool, set: BlockSet) {
+		trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}, set = {:?}", peer_id, count, h, set);
+		let mut rlp = RlpStream::new_list(4);
+		rlp.append(h);
+		rlp.append(&count);
+		rlp.append(&skip);
+		rlp.append(&if reverse {1u32} else {0u32});
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out());
+		let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed");
+		peer.asking_hash = Some(h.clone());
+		peer.block_set = Some(set);
+	}
+
+	/// Request block receipts from a peer
+	fn request_receipts(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, hashes: Vec<H256>, set: BlockSet) {
+		let mut rlp = RlpStream::new_list(hashes.len());
+		trace!(target: "sync", "{} <- GetBlockReceipts: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set);
+		for h in &hashes {
+			rlp.append(&h.clone());
+		}
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockReceipts, GET_RECEIPTS_PACKET, rlp.out());
+		let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed");
+		peer.asking_blocks = hashes;
+		peer.block_set = Some(set);
+	}
+
+	/// Request snapshot chunk from a peer.
+	fn request_snapshot_chunk(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, chunk: &H256) {
+		trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk);
+		let mut rlp = RlpStream::new_list(1);
+		rlp.append(chunk);
+		SyncRequester::send_request(sync, io, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out());
+	}
+
+	/// Generic request sender
+	fn send_request(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, asking: PeerAsking,  packet_id: PacketId, packet: Bytes) {
+		if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) {
+			if peer.asking != PeerAsking::Nothing {
+				warn!(target:"sync", "Asking {:?} while requesting {:?}", peer.asking, asking);
+			}
+			peer.asking = asking;
+			peer.ask_time = Instant::now();
+			// TODO [ToDr] This seems quite fragile. Be careful when protocol is updated.
+			let result = if packet_id >= ETH_PROTOCOL_VERSION_63.1 {
+				io.send_protocol(WARP_SYNC_PROTOCOL_ID, peer_id, packet_id, packet)
+			} else {
+				io.send(peer_id, packet_id, packet)
+			};
+			if let Err(e) = result {
+				debug!(target:"sync", "Error sending request: {:?}", e);
+				io.disconnect_peer(peer_id);
+			}
+		}
+	}
+}

ethcore/sync/src/chain/supplier.rs

@@ -0,0 +1,450 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use bytes::Bytes;
+use ethcore::client::BlockId;
+use ethcore::header::BlockNumber;
+use ethereum_types::H256;
+use network::{self, PeerId};
+use parking_lot::RwLock;
+use rlp::{Rlp, RlpStream};
+use std::cmp;
+use sync_io::SyncIo;
+
+use super::{
+	ChainSync,
+	RlpResponseResult,
+	PacketDecodeError,
+	BLOCK_BODIES_PACKET,
+	BLOCK_HEADERS_PACKET,
+	CONSENSUS_DATA_PACKET,
+	GET_BLOCK_BODIES_PACKET,
+	GET_BLOCK_HEADERS_PACKET,
+	GET_NODE_DATA_PACKET,
+	GET_RECEIPTS_PACKET,
+	GET_SNAPSHOT_DATA_PACKET,
+	GET_SNAPSHOT_MANIFEST_PACKET,
+	MAX_BODIES_TO_SEND,
+	MAX_HEADERS_TO_SEND,
+	MAX_NODE_DATA_TO_SEND,
+	MAX_RECEIPTS_HEADERS_TO_SEND,
+	MAX_RECEIPTS_TO_SEND,
+	NODE_DATA_PACKET,
+	RECEIPTS_PACKET,
+	SNAPSHOT_DATA_PACKET,
+	SNAPSHOT_MANIFEST_PACKET,
+};
+
+/// The Chain Sync Supplier: answers requests from peers with available data
+pub struct SyncSupplier;
+
+impl SyncSupplier {
+	/// Dispatch incoming requests and responses
+	pub fn dispatch_packet(sync: &RwLock<ChainSync>, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) {
+		let rlp = Rlp::new(data);
+		let result = match packet_id {
+			GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_block_bodies,
+				|e| format!("Error sending block bodies: {:?}", e)),
+
+			GET_BLOCK_HEADERS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_block_headers,
+				|e| format!("Error sending block headers: {:?}", e)),
+
+			GET_RECEIPTS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_receipts,
+				|e| format!("Error sending receipts: {:?}", e)),
+
+			GET_NODE_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_node_data,
+				|e| format!("Error sending nodes: {:?}", e)),
+
+			GET_SNAPSHOT_MANIFEST_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_snapshot_manifest,
+				|e| format!("Error sending snapshot manifest: {:?}", e)),
+
+			GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer,
+				SyncSupplier::return_snapshot_data,
+				|e| format!("Error sending snapshot data: {:?}", e)),
+			CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp),
+			_ => {
+				sync.write().on_packet(io, peer, packet_id, data);
+				Ok(())
+			}
+		};
+		result.unwrap_or_else(|e| {
+			debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e);
+		})
+	}
+
+	/// Respond to GetBlockHeaders request
+	fn return_block_headers(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		// Packet layout:
+		// [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ]
+		let max_headers: usize = r.val_at(1)?;
+		let skip: usize = r.val_at(2)?;
+		let reverse: bool = r.val_at(3)?;
+		let last = io.chain().chain_info().best_block_number;
+		let number = if r.at(0)?.size() == 32 {
+			// id is a hash
+			let hash: H256 = r.val_at(0)?;
+			trace!(target: "sync", "{} -> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", peer_id, hash, max_headers, skip, reverse);
+			match io.chain().block_header(BlockId::Hash(hash)) {
+				Some(hdr) => {
+					let number = hdr.number().into();
+					debug_assert_eq!(hdr.hash(), hash);
+
+					if max_headers == 1 || io.chain().block_hash(BlockId::Number(number)) != Some(hash) {
+						// Non canonical header or single header requested
+						// TODO: handle single-step reverse hashchains of non-canon hashes
+						trace!(target:"sync", "Returning single header: {:?}", hash);
+						let mut rlp = RlpStream::new_list(1);
+						rlp.append_raw(&hdr.into_inner(), 1);
+						return Ok(Some((BLOCK_HEADERS_PACKET, rlp)));
+					}
+					number
+				}
+				None => return Ok(Some((BLOCK_HEADERS_PACKET, RlpStream::new_list(0)))) //no such header, return nothing
+			}
+		} else {
+			let number = r.val_at::<BlockNumber>(0)?;
+			trace!(target: "sync", "{} -> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", peer_id, number, max_headers, skip, reverse);
+			number
+		};
+
+		let mut number = if reverse {
+			cmp::min(last, number)
+		} else {
+			cmp::max(0, number)
+		};
+		let max_count = cmp::min(MAX_HEADERS_TO_SEND, max_headers);
+		let mut count = 0;
+		let mut data = Bytes::new();
+		let inc = skip.saturating_add(1) as BlockNumber;
+		let overlay = io.chain_overlay().read();
+
+		// We are checking the `overlay` as well since it's where the ForkBlock
+		// header is cached : so peers can confirm we are on the right fork,
+		// even if we are not synced until the fork block
+		while (number <= last || overlay.contains_key(&number)) && count < max_count {
+			if let Some(hdr) = overlay.get(&number) {
+				trace!(target: "sync", "{}: Returning cached fork header", peer_id);
+				data.extend_from_slice(hdr);
+				count += 1;
+			} else if let Some(hdr) = io.chain().block_header(BlockId::Number(number)) {
+				data.append(&mut hdr.into_inner());
+				count += 1;
+			} else {
+				// No required block.
+				break;
+			}
+			if reverse {
+				if number <= inc || number == 0 {
+					break;
+				}
+				number = number.saturating_sub(inc);
+			} else {
+				number = number.saturating_add(inc);
+			}
+		}
+		let mut rlp = RlpStream::new_list(count as usize);
+		rlp.append_raw(&data, count as usize);
+		trace!(target: "sync", "{} -> GetBlockHeaders: returned {} entries", peer_id, count);
+		Ok(Some((BLOCK_HEADERS_PACKET, rlp)))
+	}
+
+	/// Respond to GetBlockBodies request
+	fn return_block_bodies(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		let mut count = r.item_count().unwrap_or(0);
+		if count == 0 {
+			debug!(target: "sync", "Empty GetBlockBodies request, ignoring.");
+			return Ok(None);
+		}
+		count = cmp::min(count, MAX_BODIES_TO_SEND);
+		let mut added = 0usize;
+		let mut data = Bytes::new();
+		for i in 0..count {
+			if let Some(body) = io.chain().block_body(BlockId::Hash(r.val_at::<H256>(i)?)) {
+				data.append(&mut body.into_inner());
+				added += 1;
+			}
+		}
+		let mut rlp = RlpStream::new_list(added);
+		rlp.append_raw(&data, added);
+		trace!(target: "sync", "{} -> GetBlockBodies: returned {} entries", peer_id, added);
+		Ok(Some((BLOCK_BODIES_PACKET, rlp)))
+	}
+
+	/// Respond to GetNodeData request
+	fn return_node_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		let mut count = r.item_count().unwrap_or(0);
+		trace!(target: "sync", "{} -> GetNodeData: {} entries", peer_id, count);
+		if count == 0 {
+			debug!(target: "sync", "Empty GetNodeData request, ignoring.");
+			return Ok(None);
+		}
+		count = cmp::min(count, MAX_NODE_DATA_TO_SEND);
+		let mut added = 0usize;
+		let mut data = Vec::new();
+		for i in 0..count {
+			if let Some(node) = io.chain().state_data(&r.val_at::<H256>(i)?) {
+				data.push(node);
+				added += 1;
+			}
+		}
+		trace!(target: "sync", "{} -> GetNodeData: return {} entries", peer_id, added);
+		let mut rlp = RlpStream::new_list(added);
+		for d in data {
+			rlp.append(&d);
+		}
+		Ok(Some((NODE_DATA_PACKET, rlp)))
+	}
+
+	fn return_receipts(io: &SyncIo, rlp: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		let mut count = rlp.item_count().unwrap_or(0);
+		trace!(target: "sync", "{} -> GetReceipts: {} entries", peer_id, count);
+		if count == 0 {
+			debug!(target: "sync", "Empty GetReceipts request, ignoring.");
+			return Ok(None);
+		}
+		count = cmp::min(count, MAX_RECEIPTS_HEADERS_TO_SEND);
+		let mut added_headers = 0usize;
+		let mut added_receipts = 0usize;
+		let mut data = Bytes::new();
+		for i in 0..count {
+			if let Some(mut receipts_bytes) = io.chain().block_receipts(&rlp.val_at::<H256>(i)?) {
+				data.append(&mut receipts_bytes);
+				added_receipts += receipts_bytes.len();
+				added_headers += 1;
+				if added_receipts > MAX_RECEIPTS_TO_SEND { break; }
+			}
+		}
+		let mut rlp_result = RlpStream::new_list(added_headers);
+		rlp_result.append_raw(&data, added_headers);
+		Ok(Some((RECEIPTS_PACKET, rlp_result)))
+	}
+
+	/// Respond to GetSnapshotManifest request
+	fn return_snapshot_manifest(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		let count = r.item_count().unwrap_or(0);
+		trace!(target: "warp", "{} -> GetSnapshotManifest", peer_id);
+		if count != 0 {
+			debug!(target: "warp", "Invalid GetSnapshotManifest request, ignoring.");
+			return Ok(None);
+		}
+		let rlp = match io.snapshot_service().manifest() {
+			Some(manifest) => {
+				trace!(target: "warp", "{} <- SnapshotManifest", peer_id);
+				let mut rlp = RlpStream::new_list(1);
+				rlp.append_raw(&manifest.into_rlp(), 1);
+				rlp
+			},
+			None => {
+				trace!(target: "warp", "{}: No snapshot manifest to return", peer_id);
+				RlpStream::new_list(0)
+			}
+		};
+		Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp)))
+	}
+
+	/// Respond to GetSnapshotData request
+	fn return_snapshot_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult {
+		let hash: H256 = r.val_at(0)?;
+		trace!(target: "warp", "{} -> GetSnapshotData {:?}", peer_id, hash);
+		let rlp = match io.snapshot_service().chunk(hash) {
+			Some(data) => {
+				let mut rlp = RlpStream::new_list(1);
+				trace!(target: "warp", "{} <- SnapshotData", peer_id);
+				rlp.append(&data);
+				rlp
+			},
+			None => {
+				trace!(target: "warp", "{}: No snapshot data to return", peer_id);
+				RlpStream::new_list(0)
+			}
+		};
+		Ok(Some((SNAPSHOT_DATA_PACKET, rlp)))
+	}
+
+	fn return_rlp<FRlp, FError>(io: &mut SyncIo, rlp: &Rlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError>
+		where FRlp : Fn(&SyncIo, &Rlp, PeerId) -> RlpResponseResult,
+			FError : FnOnce(network::Error) -> String
+	{
+		let response = rlp_func(io, rlp, peer);
+		match response {
+			Err(e) => Err(e),
+			Ok(Some((packet_id, rlp_stream))) => {
+				io.respond(packet_id, rlp_stream.out()).unwrap_or_else(
+					|e| debug!(target: "sync", "{:?}", error_func(e)));
+				Ok(())
+			}
+			_ => Ok(())
+		}
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use std::collections::{VecDeque};
+	use tests::helpers::{TestIo};
+	use tests::snapshot::TestSnapshotService;
+	use ethereum_types::{H256};
+	use parking_lot::RwLock;
+	use bytes::Bytes;
+	use rlp::{Rlp, RlpStream};
+	use super::{*, super::tests::*};
+	use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient};
+
+	#[test]
+	fn return_block_headers() {
+		use ethcore::views::HeaderView;
+		fn make_hash_req(h: &H256, count: usize, skip: usize, reverse: bool) -> Bytes {
+			let mut rlp = RlpStream::new_list(4);
+			rlp.append(h);
+			rlp.append(&count);
+			rlp.append(&skip);
+			rlp.append(&if reverse {1u32} else {0u32});
+			rlp.out()
+		}
+
+		fn make_num_req(n: usize, count: usize, skip: usize, reverse: bool) -> Bytes {
+			let mut rlp = RlpStream::new_list(4);
+			rlp.append(&n);
+			rlp.append(&count);
+			rlp.append(&skip);
+			rlp.append(&if reverse {1u32} else {0u32});
+			rlp.out()
+		}
+		fn to_header_vec(rlp: ::chain::RlpResponseResult) -> Vec<Bytes> {
+			Rlp::new(&rlp.unwrap().unwrap().1.out()).iter().map(|r| r.as_raw().to_vec()).collect()
+		}
+
+		let mut client = TestBlockChainClient::new();
+		client.add_blocks(100, EachBlockWith::Nothing);
+		let blocks: Vec<_> = (0 .. 100)
+			.map(|i| (&client as &BlockChainClient).block(BlockId::Number(i as BlockNumber)).map(|b| b.into_inner()).unwrap()).collect();
+		let headers: Vec<_> = blocks.iter().map(|b| Rlp::new(b).at(0).unwrap().as_raw().to_vec()).collect();
+		let hashes: Vec<_> = headers.iter().map(|h| view!(HeaderView, h).hash()).collect();
+
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let unknown: H256 = H256::new();
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, false)), 0);
+		assert!(to_header_vec(result).is_empty());
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, true)), 0);
+		assert!(to_header_vec(result).is_empty());
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, true)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[2].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, false)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[2].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, false)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, true)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, true)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[2].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, false)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[2].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, false)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]);
+
+		let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, true)), 0);
+		assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]);
+	}
+
+	#[test]
+	fn return_nodes() {
+		let mut client = TestBlockChainClient::new();
+		let queue = RwLock::new(VecDeque::new());
+		let sync = dummy_sync_with_peer(H256::new(), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let mut node_list = RlpStream::new_list(3);
+		node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
+		node_list.append(&H256::from("ffffffffffffffffffffffffffffffffffffffffffffaaaaaaaaaaaaaaaaaaaa"));
+		node_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000"));
+
+		let node_request = node_list.out();
+		// it returns rlp ONLY for hashes started with "f"
+		let result = SyncSupplier::return_node_data(&io, &Rlp::new(&node_request.clone()), 0);
+
+		assert!(result.is_ok());
+		let rlp_result = result.unwrap();
+		assert!(rlp_result.is_some());
+
+		// the length of one rlp-encoded hashe
+		let rlp = rlp_result.unwrap().1.out();
+		let rlp = Rlp::new(&rlp);
+		assert_eq!(Ok(1), rlp.item_count());
+
+		io.sender = Some(2usize);
+
+		ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request);
+		assert_eq!(1, io.packets.len());
+	}
+
+	#[test]
+	fn return_receipts_empty() {
+		let mut client = TestBlockChainClient::new();
+		let queue = RwLock::new(VecDeque::new());
+		let ss = TestSnapshotService::new();
+		let io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let result = SyncSupplier::return_receipts(&io, &Rlp::new(&[0xc0]), 0);
+
+		assert!(result.is_ok());
+	}
+
+	#[test]
+	fn return_receipts() {
+		let mut client = TestBlockChainClient::new();
+		let queue = RwLock::new(VecDeque::new());
+		let sync = dummy_sync_with_peer(H256::new(), &client);
+		let ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &ss, &queue, None);
+
+		let mut receipt_list = RlpStream::new_list(4);
+		receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
+		receipt_list.append(&H256::from("ff00000000000000000000000000000000000000000000000000000000000000"));
+		receipt_list.append(&H256::from("fff0000000000000000000000000000000000000000000000000000000000000"));
+		receipt_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000"));
+
+		let receipts_request = receipt_list.out();
+		// it returns rlp ONLY for hashes started with "f"
+		let result = SyncSupplier::return_receipts(&io, &Rlp::new(&receipts_request.clone()), 0);
+
+		assert!(result.is_ok());
+		let rlp_result = result.unwrap();
+		assert!(rlp_result.is_some());
+
+		// the length of two rlp-encoded receipts
+		assert_eq!(603, rlp_result.unwrap().1.out().len());
+
+		io.sender = Some(2usize);
+		ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request);
+		assert_eq!(1, io.packets.len());
+	}
+}

ethcore/sync/src/lib.rs

@@ -54,6 +54,8 @@ extern crate macros;
 extern crate log;
 #[macro_use]
 extern crate heapsize;
+#[macro_use]
+extern crate trace_time;
 
 mod chain;
 mod blocks;

ethcore/sync/src/light_sync/response.rs

@@ -16,13 +16,11 @@
 
 //! Helpers for decoding and verifying responses for headers.
 
-use std::fmt;
-
-use ethcore::encoded;
-use ethcore::header::Header;
+use ethcore::{encoded, header::Header};
+use ethereum_types::H256;
 use light::request::{HashOrNumber, CompleteHeadersRequest as HeadersRequest};
 use rlp::DecoderError;
-use ethereum_types::H256;
+use std::fmt;
 
 /// Errors found when decoding headers and verifying with basic constraints.
 #[derive(Debug, PartialEq)]
@@ -74,19 +72,23 @@ pub trait Constraint {
 
 /// Do basic verification of provided headers against a request.
 pub fn verify(headers: &[encoded::Header], request: &HeadersRequest) -> Result<Vec<Header>, BasicError> {
-	let headers: Vec<_> = headers.iter().map(|h| h.decode()).collect();
+	let headers: Result<Vec<_>, _> = headers.iter().map(|h| h.decode() ).collect();
+	match headers {
+		Ok(headers) => {
+			let reverse = request.reverse;
 
-	let reverse = request.reverse;
+			Max(request.max as usize).verify(&headers, reverse)?;
+			match request.start {
+				HashOrNumber::Number(ref num) => StartsAtNumber(*num).verify(&headers, reverse)?,
+				HashOrNumber::Hash(ref hash) => StartsAtHash(*hash).verify(&headers, reverse)?,
+			}
 
-	Max(request.max as usize).verify(&headers, reverse)?;
-	match request.start {
-		HashOrNumber::Number(ref num) => StartsAtNumber(*num).verify(&headers, reverse)?,
-		HashOrNumber::Hash(ref hash) => StartsAtHash(*hash).verify(&headers, reverse)?,
+			SkipsBetween(request.skip).verify(&headers, reverse)?;
+
+			Ok(headers)
+		},
+		Err(e) => Err(e.into())
 	}
-
-	SkipsBetween(request.skip).verify(&headers, reverse)?;
-
-	Ok(headers)
 }
 
 struct StartsAtNumber(u64);

ethcore/sync/src/light_sync/tests/mod.rs

@@ -45,7 +45,7 @@ fn fork_post_cht() {
 	for id in (0..CHAIN_LENGTH).map(|x| x + 1).map(BlockId::Number) {
 		let (light_peer, full_peer) = (net.peer(0), net.peer(1));
 		let light_chain = light_peer.light_chain();
-		let header = full_peer.chain().block_header(id).unwrap().decode();
+		let header = full_peer.chain().block_header(id).unwrap().decode().expect("decoding failure");
 		let _  = light_chain.import_header(header);
 		light_chain.flush_queue();
 		light_chain.import_verified();

ethcore/sync/src/snapshot.rs

@@ -14,10 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use hash::keccak;
+use ethcore::snapshot::{ManifestData, SnapshotService};
 use ethereum_types::H256;
+use hash::keccak;
+use rand::{thread_rng, Rng};
+
 use std::collections::HashSet;
-use ethcore::snapshot::ManifestData;
+use std::iter::FromIterator;
 
 #[derive(PartialEq, Eq, Debug)]
 pub enum ChunkType {
@@ -32,6 +35,7 @@ pub struct Snapshot {
 	completed_chunks: HashSet<H256>,
 	snapshot_hash: Option<H256>,
 	bad_hashes: HashSet<H256>,
+	initialized: bool,
 }
 
 impl Snapshot {
@@ -44,9 +48,29 @@ impl Snapshot {
 			completed_chunks: HashSet::new(),
 			snapshot_hash: None,
 			bad_hashes: HashSet::new(),
+			initialized: false,
 		}
 	}
 
+	/// Sync the Snapshot completed chunks with the Snapshot Service
+	pub fn initialize(&mut self, snapshot_service: &SnapshotService) {
+		if self.initialized {
+			return;
+		}
+
+		if let Some(completed_chunks) = snapshot_service.completed_chunks() {
+			self.completed_chunks = HashSet::from_iter(completed_chunks);
+		}
+
+		trace!(
+			target: "snapshot",
+			"Snapshot is now initialized with {} completed chunks.",
+			self.completed_chunks.len(),
+		);
+
+		self.initialized = true;
+	}
+
 	/// Clear everything.
 	pub fn clear(&mut self) {
 		self.pending_state_chunks.clear();
@@ -54,6 +78,7 @@ impl Snapshot {
 		self.downloading_chunks.clear();
 		self.completed_chunks.clear();
 		self.snapshot_hash = None;
+		self.initialized = false;
 	}
 
 	/// Check if currently downloading a snapshot.
@@ -89,18 +114,35 @@ impl Snapshot {
 		Err(())
 	}
 
-	/// Find a chunk to download
+	/// Find a random chunk to download
 	pub fn needed_chunk(&mut self) -> Option<H256> {
-		// check state chunks first
-		let chunk = self.pending_state_chunks.iter()
-			.chain(self.pending_block_chunks.iter())
-			.find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h))
-			.cloned();
+		// Find all random chunks: first blocks, then state
+		let needed_chunks = {
+			let chunk_filter = |h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h);
+
+			let needed_block_chunks = self.pending_block_chunks.iter()
+				.filter(|&h| chunk_filter(h))
+				.map(|h| *h)
+				.collect::<Vec<H256>>();
+
+			// If no block chunks to download, get the state chunks
+			if needed_block_chunks.len() == 0 {
+				self.pending_state_chunks.iter()
+					.filter(|&h| chunk_filter(h))
+					.map(|h| *h)
+					.collect::<Vec<H256>>()
+			} else {
+				needed_block_chunks
+			}
+		};
+
+		// Get a random chunk
+		let chunk = thread_rng().choose(&needed_chunks);
 
 		if let Some(hash) = chunk {
 			self.downloading_chunks.insert(hash.clone());
 		}
-		chunk
+		chunk.map(|h| *h)
 	}
 
 	pub fn clear_chunk_download(&mut self, hash: &H256) {
@@ -185,8 +227,15 @@ mod test {
 
 		let requested: Vec<H256> = (0..40).map(|_| snapshot.needed_chunk().unwrap()).collect();
 		assert!(snapshot.needed_chunk().is_none());
-		assert_eq!(&requested[0..20], &manifest.state_hashes[..]);
-		assert_eq!(&requested[20..40], &manifest.block_hashes[..]);
+
+		let requested_all_block_chunks = manifest.block_hashes.iter()
+			.all(|h| requested.iter().any(|rh| rh == h));
+		assert!(requested_all_block_chunks);
+
+		let requested_all_state_chunks = manifest.state_hashes.iter()
+			.all(|h| requested.iter().any(|rh| rh == h));
+		assert!(requested_all_state_chunks);
+
 		assert_eq!(snapshot.downloading_chunks.len(), 40);
 
 		assert_eq!(snapshot.validate_chunk(&state_chunks[4]), Ok(ChunkType::State(manifest.state_hashes[4].clone())));

ethcore/sync/src/tests/helpers.rs

@@ -16,13 +16,14 @@
 
 use std::collections::{VecDeque, HashSet, HashMap};
 use std::sync::Arc;
+use std::time::Duration;
 use ethereum_types::H256;
 use parking_lot::{RwLock, Mutex};
 use bytes::Bytes;
 use network::{self, PeerId, ProtocolId, PacketId, SessionInfo};
 use tests::snapshot::*;
 use ethcore::client::{TestBlockChainClient, BlockChainClient, Client as EthcoreClient,
-	ClientConfig, ChainNotify, ChainMessageType, ClientIoMessage};
+	ClientConfig, ChainNotify, ChainRoute, ChainMessageType, ClientIoMessage};
 use ethcore::header::BlockNumber;
 use ethcore::snapshot::SnapshotService;
 use ethcore::spec::Spec;
@@ -133,11 +134,11 @@ impl<'p, C> SyncIo for TestIo<'p, C> where C: FlushingBlockChainClient, C: 'p {
 	}
 
 	fn eth_protocol_version(&self, _peer: PeerId) -> u8 {
-		ETH_PROTOCOL_VERSION_63
+		ETH_PROTOCOL_VERSION_63.0
 	}
 
 	fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8 {
-		if protocol == &WARP_SYNC_PROTOCOL_ID { PAR_PROTOCOL_VERSION_3 } else { self.eth_protocol_version(peer_id) }
+		if protocol == &WARP_SYNC_PROTOCOL_ID { PAR_PROTOCOL_VERSION_3.0 } else { self.eth_protocol_version(peer_id) }
 	}
 
 	fn chain_overlay(&self) -> &RwLock<HashMap<BlockNumber, Bytes>> {
@@ -519,11 +520,9 @@ impl TestIoHandler {
 impl IoHandler<ClientIoMessage> for TestIoHandler {
 	fn message(&self, _io: &IoContext<ClientIoMessage>, net_message: &ClientIoMessage) {
 		match *net_message {
-			ClientIoMessage::NewMessage(ref message) => if let Err(e) = self.client.engine().handle_message(message) {
-				panic!("Invalid message received: {}", e);
-			},
-			ClientIoMessage::NewPrivateTransaction => {
+			ClientIoMessage::Execute(ref exec) => {
 				*self.private_tx_queued.lock() += 1;
+				(*exec.0)(&self.client);
 			},
 			_ => {} // ignore other messages
 		}
@@ -534,12 +533,13 @@ impl ChainNotify for EthPeer<EthcoreClient> {
 	fn new_blocks(&self,
 		imported: Vec<H256>,
 		invalid: Vec<H256>,
-		enacted: Vec<H256>,
-		retracted: Vec<H256>,
+		route: ChainRoute,
 		sealed: Vec<H256>,
 		proposed: Vec<Bytes>,
-		_duration: u64)
+		_duration: Duration)
 	{
+		let (enacted, retracted) = route.into_enacted_retracted();
+
 		self.new_blocks_queue.write().push_back(NewBlockMessage {
 			imported,
 			invalid,

ethcore/sync/src/tests/private.rs

@@ -24,7 +24,7 @@ use ethcore::CreateContractAddress;
 use transaction::{Transaction, Action};
 use ethcore::executive::{contract_address};
 use ethcore::test_helpers::{push_block_with_transactions};
-use ethcore_private_tx::{Provider, ProviderConfig, NoopEncryptor};
+use ethcore_private_tx::{Provider, ProviderConfig, NoopEncryptor, Importer};
 use ethcore::account_provider::AccountProvider;
 use ethkey::{KeyPair};
 use tests::helpers::{TestNet, TestIoHandler};
@@ -84,7 +84,7 @@ fn send_private_transaction() {
 			Box::new(NoopEncryptor::default()),
 			signer_config,
 			IoChannel::to_handler(Arc::downgrade(&io_handler0)),
-	).unwrap());
+	));
 	pm0.add_notify(net.peers[0].clone());
 
 	let pm1 = Arc::new(Provider::new(
@@ -94,7 +94,7 @@ fn send_private_transaction() {
 			Box::new(NoopEncryptor::default()),
 			validator_config,
 			IoChannel::to_handler(Arc::downgrade(&io_handler1)),
-	).unwrap());
+	));
 	pm1.add_notify(net.peers[1].clone());
 
 	// Create and deploy contract
@@ -133,7 +133,6 @@ fn send_private_transaction() {
 	//process received private transaction message
 	let private_transaction = received_private_transactions[0].clone();
 	assert!(pm1.import_private_transaction(&private_transaction).is_ok());
-	assert!(pm1.on_private_transaction_queued().is_ok());
 
 	//send signed response
 	net.sync();
@@ -147,4 +146,4 @@ fn send_private_transaction() {
 	assert!(pm0.import_signed_private_transaction(&signed_private_transaction).is_ok());
 	let local_transactions = net.peer(0).miner.local_transactions();
 	assert_eq!(local_transactions.len(), 1);
-}
+}

ethcore/sync/src/tests/snapshot.rs

@@ -22,7 +22,7 @@ use parking_lot::Mutex;
 use bytes::Bytes;
 use ethcore::snapshot::{SnapshotService, ManifestData, RestorationStatus};
 use ethcore::header::BlockNumber;
-use ethcore::client::{EachBlockWith};
+use ethcore::client::EachBlockWith;
 use super::helpers::*;
 use {SyncConfig, WarpSync};
 
@@ -80,6 +80,10 @@ impl SnapshotService for TestSnapshotService {
 		Some((1, 2))
 	}
 
+	fn completed_chunks(&self) -> Option<Vec<H256>> {
+		Some(vec![])
+	}
+
 	fn chunk(&self, hash: H256) -> Option<Bytes> {
 		self.chunks.get(&hash).cloned()
 	}
@@ -99,7 +103,15 @@ impl SnapshotService for TestSnapshotService {
 	}
 
 	fn begin_restore(&self, manifest: ManifestData) {
-		*self.restoration_manifest.lock() = Some(manifest);
+		let mut restoration_manifest = self.restoration_manifest.lock();
+
+		if let Some(ref c_manifest) = *restoration_manifest {
+			if c_manifest.state_root == manifest.state_root {
+				return;
+			}
+		}
+
+		*restoration_manifest = Some(manifest);
 		self.state_restoration_chunks.lock().clear();
 		self.block_restoration_chunks.lock().clear();
 	}
@@ -121,6 +133,10 @@ impl SnapshotService for TestSnapshotService {
 			self.block_restoration_chunks.lock().insert(hash, chunk);
 		}
 	}
+
+	fn shutdown(&self) {
+		self.abort_restore();
+	}
 }
 
 #[test]

ethcore/transaction/src/error.rs

@@ -18,6 +18,7 @@ use std::{fmt, error};
 
 use ethereum_types::U256;
 use ethkey;
+use rlp;
 use unexpected::OutOfBounds;
 
 #[derive(Debug, PartialEq, Clone)]
@@ -74,6 +75,10 @@ pub enum Error {
 	NotAllowed,
 	/// Signature error
 	InvalidSignature(String),
+	/// Transaction too big
+	TooBig,
+	/// Invalid RLP encoding
+	InvalidRlp(String),
 }
 
 impl From<ethkey::Error> for Error {
@@ -82,6 +87,12 @@ impl From<ethkey::Error> for Error {
 	}
 }
 
+impl From<rlp::DecoderError> for Error {
+	fn from(err: rlp::DecoderError) -> Self {
+		Error::InvalidRlp(format!("{}", err))
+	}
+}
+
 impl fmt::Display for Error {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		use self::Error::*;
@@ -106,6 +117,8 @@ impl fmt::Display for Error {
 			InvalidChainId => "Transaction of this chain ID is not allowed on this chain.".into(),
 			InvalidSignature(ref err) => format!("Transaction has invalid signature: {}.", err),
 			NotAllowed => "Sender does not have permissions to execute this type of transction".into(),
+			TooBig => "Transaction too big".into(),
+			InvalidRlp(ref err) => format!("Transaction has invalid RLP structure: {}.", err),
 		};
 
 		f.write_fmt(format_args!("Transaction error ({})", msg))

ethcore/transaction/src/transaction.rs

@@ -409,6 +409,10 @@ impl UnverifiedTransaction {
 		if check_low_s && !(allow_empty_signature && self.is_unsigned()) {
 			self.check_low_s()?;
 		}
+		// Disallow unsigned transactions in case EIP-86 is disabled.
+		if !allow_empty_signature && self.is_unsigned() {
+			return Err(ethkey::Error::InvalidSignature.into());
+		}
 		// EIP-86: Transactions of this form MUST have gasprice = 0, nonce = 0, value = 0, and do NOT increment the nonce of account 0.
 		if allow_empty_signature && self.is_unsigned() && !(self.gas_price.is_zero() && self.value.is_zero() && self.nonce.is_zero()) {
 			return Err(ethkey::Error::InvalidSignature.into())
@@ -576,7 +580,8 @@ mod tests {
 
 	#[test]
 	fn sender_test() {
-		let t: UnverifiedTransaction = rlp::decode(&::rustc_hex::FromHex::from_hex("f85f800182520894095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba048b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353a0efffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804").unwrap());
+		let bytes = ::rustc_hex::FromHex::from_hex("f85f800182520894095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba048b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353a0efffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804").unwrap();
+		let t: UnverifiedTransaction = rlp::decode(&bytes).expect("decoding UnverifiedTransaction failed");
 		assert_eq!(t.data, b"");
 		assert_eq!(t.gas, U256::from(0x5208u64));
 		assert_eq!(t.gas_price, U256::from(0x01u64));
@@ -645,7 +650,7 @@ mod tests {
 		use rustc_hex::FromHex;
 
 		let test_vector = |tx_data: &str, address: &'static str| {
-			let signed = rlp::decode(&FromHex::from_hex(tx_data).unwrap());
+			let signed = rlp::decode(&FromHex::from_hex(tx_data).unwrap()).expect("decoding tx data failed");
 			let signed = SignedTransaction::new(signed).unwrap();
 			assert_eq!(signed.sender(), address.into());
 			println!("chainid: {:?}", signed.chain_id());

ethcore/types/src/receipt.rs

@@ -193,7 +193,7 @@ mod tests {
 		);
 		let encoded = ::rlp::encode(&r);
 		assert_eq!(&encoded[..], &expected[..]);
-		let decoded: Receipt = ::rlp::decode(&encoded);
+		let decoded: Receipt = ::rlp::decode(&encoded).expect("decoding receipt failed");
 		assert_eq!(decoded, r);
 	}
 
@@ -211,7 +211,7 @@ mod tests {
 		);
 		let encoded = ::rlp::encode(&r);
 		assert_eq!(&encoded[..], &expected[..]);
-		let decoded: Receipt = ::rlp::decode(&encoded);
+		let decoded: Receipt = ::rlp::decode(&encoded).expect("decoding receipt failed");
 		assert_eq!(decoded, r);
 	}
 }

ethcore/types/src/restoration_status.rs

@@ -21,6 +21,11 @@
 pub enum RestorationStatus {
 	///	No restoration.
 	Inactive,
+	/// Restoration is initalizing
+	Initializing {
+		/// Number of chunks done/imported
+		chunks_done: u32,
+	},
 	/// Ongoing restoration.
 	Ongoing {
 		/// Total number of state chunks.

ethcore/vm/src/call_type.rs

@@ -64,7 +64,7 @@ mod tests {
 	fn should_encode_and_decode_call_type() {
 		let original = CallType::Call;
 		let encoded = encode(&original);
-		let decoded = decode(&encoded);
+		let decoded = decode(&encoded).expect("failure decoding CallType");
 		assert_eq!(original, decoded);
 	}
 }

ethcore/wasm/Cargo.toml

@@ -12,4 +12,4 @@ libc = "0.2"
 pwasm-utils = "0.1"
 vm = { path = "../vm" }
 ethcore-logger = { path = "../../logger" }
-wasmi = { version = "0.1.3", features = ["opt-in-32bit"] }
+wasmi = { version = "0.2" }

json/src/spec/params.rs

@@ -113,6 +113,9 @@ pub struct Params {
 	/// See main EthashParams docs.
 	#[serde(rename="maxCodeSize")]
 	pub max_code_size: Option<Uint>,
+	/// Maximum size of transaction RLP payload.
+	#[serde(rename="maxTransactionSize")]
+	pub max_transaction_size: Option<Uint>,
 	/// See main EthashParams docs.
 	#[serde(rename="maxCodeSizeTransition")]
 	pub max_code_size_transition: Option<Uint>,

mac/Parity.pkgproj

@@ -462,7 +462,7 @@
 				<key>OVERWRITE_PERMISSIONS</key>
 				<false/>
 				<key>VERSION</key>
-				<string>1.11.0</string>
+				<string>1.11.5</string>
 			</dict>
 			<key>UUID</key>
 			<string>2DCD5B81-7BAF-4DA1-9251-6274B089FD36</string>

mac/post-install.sh

@@ -3,6 +3,4 @@
 test -f /usr/local/libexec/uninstall-parity.sh && /usr/local/libexec/uninstall-parity.sh || true
 killall -9 parity && sleep 5
 su $USER -c "open /Applications/Parity\ Ethereum.app"
-sleep 5
-su $USER -c "open http://127.0.0.1:8180/"
 exit 0

miner/Cargo.toml

@@ -27,7 +27,7 @@ linked-hash-map = "0.5"
 log = "0.3"
 parking_lot = "0.5"
 price-info = { path = "../price-info" }
-rayon = "1.0"
+rlp = { path = "../util/rlp" }
 trace-time = { path = "../util/trace-time" }
 transaction-pool = { path = "../transaction-pool" }
 

miner/src/lib.rs

@@ -29,7 +29,7 @@ extern crate keccak_hash as hash;
 extern crate linked_hash_map;
 extern crate parking_lot;
 extern crate price_info;
-extern crate rayon;
+extern crate rlp;
 extern crate trace_time;
 extern crate transaction_pool as txpool;
 

miner/src/pool/client.rs

@@ -62,6 +62,10 @@ pub trait Client: fmt::Debug + Sync {
 
 	/// Classify transaction (check if transaction is filtered by some contracts).
 	fn transaction_type(&self, tx: &transaction::SignedTransaction) -> TransactionType;
+
+	/// Performs pre-validation of RLP decoded transaction
+	fn decode_transaction(&self, transaction: &[u8])
+		-> Result<transaction::UnverifiedTransaction, transaction::Error>;
 }
 
 /// State nonce client

miner/src/pool/queue.rs

@@ -23,7 +23,6 @@ use std::collections::BTreeMap;
 
 use ethereum_types::{H256, U256, Address};
 use parking_lot::RwLock;
-use rayon::prelude::*;
 use transaction;
 use txpool::{self, Verifier};
 
@@ -174,13 +173,19 @@ impl TransactionQueue {
 		transactions: Vec<verifier::Transaction>,
 	) -> Vec<Result<(), transaction::Error>> {
 		// Run verification
-		let _timer = ::trace_time::PerfTimer::new("queue::verifyAndImport");
+		let _timer = ::trace_time::PerfTimer::new("pool::verify_and_import");
 		let options = self.options.read().clone();
 
 		let verifier = verifier::Verifier::new(client, options, self.insertion_id.clone());
 		let results = transactions
-			.into_par_iter()
-			.map(|transaction| verifier.verify_transaction(transaction))
+			.into_iter()
+			.map(|transaction| {
+				if self.pool.read().find(&transaction.hash()).is_some() {
+					bail!(transaction::Error::AlreadyImported)
+				}
+
+				verifier.verify_transaction(transaction)
+			})
 			.map(|result| result.and_then(|verified| {
 				self.pool.write().import(verified)
 					.map(|_imported| ())

miner/src/pool/scoring.rs

@@ -108,6 +108,15 @@ impl txpool::Scoring<VerifiedTransaction> for NonceAndGasPrice {
 			}
 		}
 
+		// Always kick out non-local transactions in favour of local ones.
+		if new.priority().is_local() && !old.priority().is_local() {
+			return true;
+		}
+		// And never kick out local transactions in favour of external ones.
+		if !new.priority().is_local() && old.priority.is_local() {
+			return false;
+		}
+
 		self.choose(old, new) == txpool::scoring::Choice::ReplaceOld
 	}
 }
@@ -119,6 +128,21 @@ mod tests {
 	use pool::tests::tx::{Tx, TxExt};
 	use txpool::Scoring;
 
+	#[test]
+	fn should_replace_non_local_transaction_with_local_one() {
+		// given
+		let scoring = NonceAndGasPrice(PrioritizationStrategy::GasPriceOnly);
+		let tx1 = Tx::default().signed().verified();
+		let tx2 = {
+			let mut tx = Tx::default().signed().verified();
+			tx.priority = ::pool::Priority::Local;
+			tx
+		};
+
+		assert!(scoring.should_replace(&tx1, &tx2));
+		assert!(!scoring.should_replace(&tx2, &tx1));
+	}
+
 	#[test]
 	fn should_calculate_score_correctly() {
 		// given

miner/src/pool/tests/client.rs

@@ -14,23 +14,31 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::sync::{atomic, Arc};
+
 use ethereum_types::{U256, H256, Address};
+use rlp::Rlp;
 use transaction::{self, Transaction, SignedTransaction, UnverifiedTransaction};
 
 use pool;
 use pool::client::AccountDetails;
 
+const MAX_TRANSACTION_SIZE: usize = 15 * 1024;
+
 #[derive(Debug, Clone)]
 pub struct TestClient {
+	verification_invoked: Arc<atomic::AtomicBool>,
 	account_details: AccountDetails,
 	gas_required: U256,
 	is_service_transaction: bool,
 	local_address: Address,
+	max_transaction_size: usize,
 }
 
 impl Default for TestClient {
 	fn default() -> Self {
 		TestClient {
+			verification_invoked: Default::default(),
 			account_details: AccountDetails {
 				nonce: 123.into(),
 				balance: 63_100.into(),
@@ -39,6 +47,7 @@ impl Default for TestClient {
 			gas_required: 21_000.into(),
 			is_service_transaction: false,
 			local_address: Default::default(),
+			max_transaction_size: MAX_TRANSACTION_SIZE,
 		}
 	}
 }
@@ -83,6 +92,10 @@ impl TestClient {
 			insertion_id: 1,
 		}
 	}
+
+	pub fn was_verification_triggered(&self) -> bool {
+		self.verification_invoked.load(atomic::Ordering::SeqCst)
+	}
 }
 
 impl pool::client::Client for TestClient {
@@ -93,6 +106,7 @@ impl pool::client::Client for TestClient {
 	fn verify_transaction(&self, tx: UnverifiedTransaction)
 		-> Result<SignedTransaction, transaction::Error>
 	{
+		self.verification_invoked.store(true, atomic::Ordering::SeqCst);
 		Ok(SignedTransaction::new(tx)?)
 	}
 
@@ -116,6 +130,15 @@ impl pool::client::Client for TestClient {
 			pool::client::TransactionType::Regular
 		}
 	}
+
+	fn decode_transaction(&self, transaction: &[u8]) -> Result<UnverifiedTransaction, transaction::Error> {
+		let rlp = Rlp::new(&transaction);
+		if rlp.as_raw().len() > self.max_transaction_size {
+			return Err(transaction::Error::TooBig)
+		}
+		rlp.as_val().map_err(|e| transaction::Error::InvalidRlp(e.to_string()))
+	}
+
 }
 
 impl pool::client::NonceClient for TestClient {

miner/src/pool/tests/mod.rs

@@ -63,8 +63,10 @@ fn should_return_correct_nonces_when_dropped_because_of_limit() {
 	let nonce = tx1.nonce;
 
 	// when
-	let result = txq.import(TestClient::new(), vec![tx1, tx2].local());
-	assert_eq!(result, vec![Ok(()), Err(transaction::Error::LimitReached)]);
+	let r1= txq.import(TestClient::new(), vec![tx1].local());
+	let r2= txq.import(TestClient::new(), vec![tx2].local());
+	assert_eq!(r1, vec![Ok(())]);
+	assert_eq!(r2, vec![Err(transaction::Error::LimitReached)]);
 	assert_eq!(txq.status().status.transaction_count, 1);
 
 	// then
@@ -755,3 +757,78 @@ fn should_clear_cache_after_timeout_for_local() {
 	// then
 	assert_eq!(txq.pending(TestClient::new(), 0, 1002, None).len(), 2);
 }
+
+#[test]
+fn should_reject_big_transaction() {
+	let txq = new_queue();
+	let big_tx = Tx::default().big_one();
+	let res = txq.import(TestClient::new(), vec![
+		verifier::Transaction::Local(PendingTransaction::new(big_tx, transaction::Condition::Timestamp(1000).into()))
+	]);
+	assert_eq!(res, vec![Err(transaction::Error::TooBig)]);
+}
+
+#[test]
+fn should_include_local_transaction_to_a_full_pool() {
+	// given
+	let txq = TransactionQueue::new(
+		txpool::Options {
+			max_count: 1,
+			max_per_sender: 2,
+			max_mem_usage: 50
+		},
+		verifier::Options {
+			minimal_gas_price: 1.into(),
+			block_gas_limit: 1_000_000.into(),
+			tx_gas_limit: 1_000_000.into(),
+		},
+		PrioritizationStrategy::GasPriceOnly,
+	);
+	let tx1 = Tx::gas_price(10_000).signed().unverified();
+	let tx2 = Tx::gas_price(1).signed().local();
+
+	let res = txq.import(TestClient::new().with_balance(1_000_000_000), vec![tx1]);
+	assert_eq!(res, vec![Ok(())]);
+	assert_eq!(txq.status().status.transaction_count, 1);
+
+	// when
+	let res = txq.import(TestClient::new(), vec![tx2]);
+	assert_eq!(res, vec![Ok(())]);
+
+	// then
+	assert_eq!(txq.status().status.transaction_count, 1);
+}
+
+#[test]
+fn should_avoid_verifying_transaction_already_in_pool() {
+	// given
+	let txq = TransactionQueue::new(
+		txpool::Options {
+			max_count: 1,
+			max_per_sender: 2,
+			max_mem_usage: 50
+		},
+		verifier::Options {
+			minimal_gas_price: 1.into(),
+			block_gas_limit: 1_000_000.into(),
+			tx_gas_limit: 1_000_000.into(),
+		},
+		PrioritizationStrategy::GasPriceOnly,
+	);
+	let client = TestClient::new();
+	let tx1 = Tx::default().signed().unverified();
+
+	let res = txq.import(client.clone(), vec![tx1.clone()]);
+	assert_eq!(res, vec![Ok(())]);
+	assert_eq!(txq.status().status.transaction_count, 1);
+	assert!(client.was_verification_triggered());
+
+	// when
+	let client = TestClient::new();
+	let res = txq.import(client.clone(), vec![tx1]);
+	assert_eq!(res, vec![Err(transaction::Error::AlreadyImported)]);
+	assert!(!client.was_verification_triggered());
+
+	// then
+	assert_eq!(txq.status().status.transaction_count, 1);
+}

miner/src/pool/tests/tx.rs

@@ -87,6 +87,19 @@ impl Tx {
 			nonce: self.nonce.into()
 		}
 	}
+
+	pub fn big_one(self) -> SignedTransaction {
+		let keypair = Random.generate().unwrap();
+		let tx = Transaction {
+			action: transaction::Action::Create,
+			value: U256::from(100),
+			data: include_str!("../res/big_transaction.data").from_hex().unwrap(),
+			gas: self.gas.into(),
+			gas_price: self.gas_price.into(),
+			nonce: self.nonce.into()
+		};
+		tx.sign(keypair.secret(), None)
+	}
 }
 pub trait TxExt: Sized {
 	type Out;

miner/src/pool/verifier.rs

@@ -27,6 +27,7 @@ use std::sync::Arc;
 use std::sync::atomic::{self, AtomicUsize};
 
 use ethereum_types::{U256, H256};
+use rlp::Encodable;
 use transaction;
 use txpool;
 
@@ -56,6 +57,7 @@ impl Default for Options {
 }
 
 /// Transaction to verify.
+#[cfg_attr(test, derive(Clone))]
 pub enum Transaction {
 	/// Fresh, never verified transaction.
 	///
@@ -74,7 +76,8 @@ pub enum Transaction {
 }
 
 impl Transaction {
-	fn hash(&self) -> H256 {
+	/// Return transaction hash
+	pub fn hash(&self) -> H256 {
 		match *self {
 			Transaction::Unverified(ref tx) => tx.hash(),
 			Transaction::Retracted(ref tx) => tx.hash(),
@@ -222,6 +225,12 @@ impl<C: Client> txpool::Verifier<Transaction> for Verifier<C> {
 			Transaction::Local(tx) => tx,
 		};
 
+		// Verify RLP payload
+		if let Err(err) = self.client.decode_transaction(&transaction.rlp_bytes()) {
+			debug!(target: "txqueue", "[{:?}] Rejected transaction's rlp payload", err);
+			bail!(err)
+		}
+
 		let sender = transaction.sender();
 		let account_details = self.client.account_details(&sender);
 

nsis/installer.nsi

@@ -10,7 +10,7 @@
 !define DESCRIPTION "Fast, light, robust Ethereum implementation"
 !define VERSIONMAJOR 1
 !define VERSIONMINOR 11
-!define VERSIONBUILD 0
+!define VERSIONBUILD 5
 !define ARGS ""
 !define FIRST_START_ARGS "--mode=passive ui"