v2.6.5 beta (#11240)

* [CI] check evmbin build (#11096)
* Correct EIP-712 encoding (#11092)
* [client]: Fix for incorrectly dropped consensus messages (#11082) (#11086)
* Update hardcoded headers (foundation, classic, kovan, xdai, ewc, ...) (#11053)
* Add cargo-remote dir to .gitignore (?)
* Update light client headers: ropsten 6631425 foundation 8798209 (#11201)
* Update list of bootnodes for xDai chain (#11236)
* ethcore/res: add mordor testnet configuration (#11200)
* [chain specs]: activate Istanbul on mainnet (#11228)
* [builtin]: support multiple prices and activations in chain spec (#11039)
* [receipt]: add sender & receiver to RichReceipts (#11179)
* [ethcore/builtin]: do not panic in blake2pricer on short input (#11180)
* Made ecrecover implementation trait public (#11188)
* Fix docker centos build (#11226)
* Update MIX bootnodes. (#11203)
* Insert explicit warning into the panic hook (#11225)
* Use provided usd-per-eth value if an endpoint is specified (#11209)
* Cleanup stratum a bit (#11161)
* Add Constantinople EIPs to the dev (instant_seal) config (#10809) (already backported)
* util Host: fix a double Read Lock bug in fn Host::session_readable() (#11175)
* ethcore client: fix a double Read Lock bug in fn Client::logs() (#11172)
* Type annotation for next_key() matching of json filter options (#11192)
* Upgrade jsonrpc to latest (#11206)
* [dependencies]: jsonrpc 14.0.1 (#11183)
* Upgrade to jsonrpc v14 (#11151)
* Switching sccache from local to Redis (#10971)
* Snapshot restoration overhaul (#11219)
* Add new line after writing block to hex file. (#10984)
* Pause pruning while snapshotting (#11178)
* Change how RPCs eth_call and eth_estimateGas handle "Pending" (#11127)
* Fix block detail updating (#11015)
* Make InstantSeal Instant again #11186
* Filter out some bad ropsten warp snapshots (#11247)

ethcore/src/client/client.rs

@@ -15,77 +15,99 @@
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::cmp;
-use std::collections::{HashSet, BTreeMap, VecDeque};
-use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
+use std::collections::{BTreeMap, HashSet, VecDeque};
+use std::convert::TryFrom;
+use std::io::{BufRead, BufReader};
+use std::str::from_utf8;
 use std::sync::{Arc, Weak};
-use std::time::{Instant, Duration};
+use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering, Ordering, AtomicU64};
+use std::time::{Duration, Instant};
 
-use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert, BlockNumberKey};
+use ansi_term::Colour;
 use bytes::Bytes;
-use call_contract::{CallContract, RegistryInfo};
-use ethcore_miner::pool::VerifiedTransaction;
-use ethereum_types::{H256, H264, Address, U256};
-use evm::Schedule;
+use bytes::ToPretty;
+use ethereum_types::{Address, H256, H264, U256};
 use hash::keccak;
-use io::IoChannel;
+use hash_db::EMPTY_PREFIX;
 use itertools::Itertools;
-use journaldb;
-use kvdb::{DBValue, KeyValueDB, DBTransaction};
+use kvdb::{DBTransaction, DBValue, KeyValueDB};
 use parking_lot::{Mutex, RwLock};
 use rand::rngs::OsRng;
-use types::transaction::{self, LocalizedTransaction, UnverifiedTransaction, SignedTransaction, Action};
-use trie::{TrieSpec, TrieFactory, Trie};
-use types::ancestry_action::AncestryAction;
-use types::encoded;
-use types::filter::Filter;
-use types::log_entry::LocalizedLogEntry;
-use types::receipt::{Receipt, LocalizedReceipt};
-use types::{BlockNumber, header::{Header, ExtendedHeader}};
-use vm::{EnvInfo, LastHashes};
-use hash_db::EMPTY_PREFIX;
-use block::{LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock};
-use client::ancient_import::AncientVerifier;
+use rlp::PayloadInfo;
+use rustc_hex::FromHex;
+use trie::{Trie, TrieFactory, TrieSpec};
+
+use block::{ClosedBlock, Drain, enact_verified, LockedBlock, OpenBlock, SealedBlock};
+use blockchain::{
+	BlockChain,
+	BlockChainDB,
+	BlockNumberKey,
+	BlockProvider,
+	BlockReceipts,
+	ExtrasInsert,
+	ImportRoute,
+	TransactionAddress,
+	TreeRoute
+};
+use call_contract::{CallContract, RegistryInfo};
+use client::bad_blocks;
 use client::{
 	Nonce, Balance, ChainInfo, BlockInfo, TransactionInfo,
 	ReopenBlock, PrepareOpenBlock, ScheduleInfo, ImportSealedBlock,
 	BroadcastProposalBlock, ImportBlock, StateOrBlock, StateInfo, StateClient, Call,
 	AccountData, BlockChain as BlockChainTrait, BlockProducer, SealedBlockImporter,
-	ClientIoMessage, BlockChainReset
+	ClientIoMessage, BlockChainReset, ImportExportBlocks, ForceUpdateSealing
 };
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
 	TraceFilter, CallAnalytics, Mode,
 	ChainNotify, NewBlocks, ChainRoute, PruningInfo, ProvingBlockChainClient, EngineInfo, ChainMessageType,
-	IoClient, BadBlocks,
+	IoClient, BadBlocks
 };
-use client::bad_blocks;
+use client::ancient_import::AncientVerifier;
+use db::{keys::BlockDetails, Readable, Writable};
 use engines::{MAX_UNCLE_AGE, Engine, EpochTransition, ForkChoice, EngineError, SealingState};
 use engines::epoch::PendingTransition;
 use error::{
 	ImportError, ExecutionError, CallError, BlockError,
-	QueueError, Error as EthcoreError, EthcoreResult,
+	QueueError, Error as EthcoreError, EthcoreResult
 };
+use ethcore_miner::pool::VerifiedTransaction;
+use evm::Schedule;
 use executive::{Executive, Executed, TransactOptions, contract_address};
 use factory::{Factories, VmFactory};
+use io::IoChannel;
+use journaldb;
 use miner::{Miner, MinerService};
-use snapshot::{self, io as snapshot_io, SnapshotClient};
+use snapshot::{self, io as snapshot_io, SnapshotClient, Error as SnapshotError, Progress};
+use snapshot::io::SnapshotWriter;
 use spec::Spec;
 use state::{self, State};
 use state_db::StateDB;
-use trace::{self, TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase};
+use trace::{self, Database as TraceDatabase, ImportRequest as TraceImportRequest, LocalizedTrace, TraceDB};
 use transaction_ext::Transaction;
-use verification::queue::kind::BlockLike;
-use verification::queue::kind::blocks::Unverified;
-use verification::{PreverifiedBlock, Verifier, BlockQueue};
+use types::{
+	ancestry_action::AncestryAction,
+	BlockNumber,
+	encoded,
+	data_format::DataFormat,
+	filter::Filter,
+	header::{Header, ExtendedHeader},
+	log_entry::LocalizedLogEntry,
+	receipt::{LocalizedReceipt, Receipt},
+	transaction::{self, Action, LocalizedTransaction, SignedTransaction, UnverifiedTransaction},
+};
 use verification;
-use ansi_term::Colour;
+use verification::{BlockQueue, Verifier, PreverifiedBlock};
+use verification::queue::kind::blocks::Unverified;
+use verification::queue::kind::BlockLike;
+use vm::{CreateContractAddress, EnvInfo, LastHashes};
 
 // re-export
 pub use types::blockchain_info::BlockChainInfo;
 pub use types::block_status::BlockStatus;
 pub use blockchain::CacheSize as BlockChainCacheSize;
 pub use verification::QueueInfo as BlockQueueInfo;
-use db::{Writable, Readable, keys::BlockDetails};
 
 use_contract!(registry, "res/contracts/registrar.json");
 
@@ -194,6 +216,9 @@ pub struct Client {
 	/// Database pruning strategy to use for StateDB
 	pruning: journaldb::Algorithm,
 
+	/// Don't prune the state we're currently snapshotting
+	snapshotting_at: AtomicU64,
+
 	/// Client uses this to store blocks, traces, etc.
 	db: RwLock<Arc<dyn BlockChainDB>>,
 
@@ -773,6 +798,7 @@ impl Client {
 			tracedb,
 			engine,
 			pruning: config.pruning.clone(),
+			snapshotting_at: AtomicU64::new(0),
 			db: RwLock::new(db.clone()),
 			state_db: RwLock::new(state_db),
 			report: RwLock::new(Default::default()),
@@ -957,31 +983,47 @@ impl Client {
 	}
 
 	// prune ancient states until below the memory limit or only the minimum amount remain.
-	fn prune_ancient(&self, mut state_db: StateDB, chain: &BlockChain) -> Result<(), ::error::Error> {
-		let number = match state_db.journal_db().latest_era() {
+	fn prune_ancient(&self, mut state_db: StateDB, chain: &BlockChain) -> Result<(), EthcoreError> {
+		if !state_db.journal_db().is_pruned() {
+			return Ok(())
+		}
+
+		let latest_era = match state_db.journal_db().latest_era() {
 			Some(n) => n,
 			None => return Ok(()),
 		};
 
-		// prune all ancient eras until we're below the memory target,
-		// but have at least the minimum number of states.
+		// Prune all ancient eras until we're below the memory target (default: 32Mb),
+		// but have at least the minimum number of states, i.e. `history`.
+		// If a snapshot is under way, no pruning happens and memory consumption is allowed to
+		// increase above the memory target until the snapshot has finished.
 		loop {
 			let needs_pruning = state_db.journal_db().is_pruned() &&
 				state_db.journal_db().journal_size() >= self.config.history_mem;
 
 			if !needs_pruning { break }
 			match state_db.journal_db().earliest_era() {
-				Some(era) if era + self.history <= number => {
-					trace!(target: "client", "Pruning state for ancient era {}", era);
-					match chain.block_hash(era) {
+				Some(earliest_era) if earliest_era + self.history <= latest_era => {
+					let freeze_at = self.snapshotting_at.load(Ordering::SeqCst);
+					if freeze_at > 0 && freeze_at == earliest_era {
+						// Note: journal_db().mem_used() can be used for a more accurate memory
+						// consumption measurement but it can be expensive so sticking with the
+						// faster `journal_size()` instead.
+						trace!(target: "pruning", "Pruning is paused at era {} (snapshot under way); earliest era={}, latest era={}, journal_size={} – Not pruning.",
+						       freeze_at, earliest_era, latest_era, state_db.journal_db().journal_size());
+						break;
+					}
+					trace!(target: "pruning", "Pruning state for ancient era #{}; latest era={}, journal_size={}",
+					       earliest_era, latest_era, state_db.journal_db().journal_size());
+					match chain.block_hash(earliest_era) {
 						Some(ancient_hash) => {
 							let mut batch = DBTransaction::new();
-							state_db.mark_canonical(&mut batch, era, &ancient_hash)?;
+							state_db.mark_canonical(&mut batch, earliest_era, &ancient_hash)?;
 							self.db.read().key_value().write_buffered(batch);
 							state_db.journal_db().flush();
 						}
 						None =>
-							debug!(target: "client", "Missing expected hash for block {}", era),
+							debug!(target: "pruning", "Missing expected hash for block {}", earliest_era),
 					}
 				}
 				_ => break, // means that every era is kept, no pruning necessary.
@@ -1023,15 +1065,16 @@ impl Client {
 	}
 
 	/// Get a copy of the best block's state.
-	pub fn latest_state(&self) -> State<StateDB> {
+	pub fn latest_state_and_header(&self) -> (State<StateDB>, Header) {
 		let header = self.best_block_header();
-		State::from_existing(
+		let state = State::from_existing(
 			self.state_db.read().boxed_clone_canon(&header.hash()),
 			*header.state_root(),
 			self.engine.account_start_nonce(header.number()),
 			self.factories.clone()
 		)
-		.expect("State root of best block header always valid.")
+		.expect("State root of best block header always valid.");
+		(state, header)
 	}
 
 	/// Attempt to get a copy of a specific block's final state.
@@ -1041,9 +1084,9 @@ impl Client {
 	/// is unknown.
 	pub fn state_at(&self, id: BlockId) -> Option<State<StateDB>> {
 		// fast path for latest state.
-		match id.clone() {
-			BlockId::Latest => return Some(self.latest_state()),
-			_ => {},
+		if let BlockId::Latest = id {
+			let (state, _) = self.latest_state_and_header();
+			return Some(state)
 		}
 
 		let block_number = match self.block_number(id) {
@@ -1077,8 +1120,9 @@ impl Client {
 	}
 
 	/// Get a copy of the best block's state.
-	pub fn state(&self) -> Box<dyn StateInfo> {
-		Box::new(self.latest_state()) as Box<_>
+	pub fn state(&self) -> impl StateInfo {
+		let (state, _) = self.latest_state_and_header();
+		state
 	}
 
 	/// Get info on the cache.
@@ -1531,8 +1575,8 @@ impl ImportBlock for Client {
 impl StateClient for Client {
 	type State = State<::state_db::StateDB>;
 
-	fn latest_state(&self) -> Self::State {
-		Client::latest_state(self)
+	fn latest_state_and_header(&self) -> (Self::State, Header) {
+		Client::latest_state_and_header(self)
 	}
 
 	fn state_at(&self, id: BlockId) -> Option<Self::State> {
@@ -1749,6 +1793,10 @@ impl BlockChainClient for Client {
 		self.config.spec_name.clone()
 	}
 
+	fn chain(&self) -> Arc<BlockProvider> {
+		self.chain.read().clone()
+	}
+
 	fn set_spec_name(&self, new_spec_name: String) -> Result<(), ()> {
 		trace!(target: "mode", "Client::set_spec_name({:?})", new_spec_name);
 		if !self.enabled.load(AtomicOrdering::Relaxed) {
@@ -2081,7 +2129,7 @@ impl BlockChainClient for Client {
 			blocks
 		};
 
-		Ok(self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit))
+		Ok(chain.logs(blocks, |entry| filter.matches(entry), filter.limit))
 	}
 
 	fn filter_traces(&self, filter: TraceFilter) -> Option<Vec<LocalizedTrace>> {
@@ -2402,7 +2450,9 @@ impl ImportSealedBlock for Client {
 		let raw = block.rlp_bytes();
 		let header = block.header.clone();
 		let hash = header.hash();
-		self.notify(|n| n.block_pre_import(&raw, &hash, header.difficulty()));
+		self.notify(|n| {
+			n.block_pre_import(&raw, &hash, header.difficulty())
+		});
 
 		let route = {
 			// Do a super duper basic verification to detect potential bugs
@@ -2490,19 +2540,22 @@ impl ::miner::TransactionVerifierClient for Client {}
 impl ::miner::BlockChainClient for Client {}
 
 impl super::traits::EngineClient for Client {
-	fn update_sealing(&self) {
-		self.importer.miner.update_sealing(self)
+	fn update_sealing(&self, force: ForceUpdateSealing) {
+		self.importer.miner.update_sealing(self, force)
 	}
 
 	fn submit_seal(&self, block_hash: H256, seal: Vec<Bytes>) {
-		let import = self.importer.miner.submit_seal(block_hash, seal).and_then(|block| self.import_sealed_block(block));
+		let import = self.importer.miner.submit_seal(block_hash, seal)
+			.and_then(|block| self.import_sealed_block(block));
 		if let Err(err) = import {
 			warn!(target: "poa", "Wrong internal seal submission! {:?}", err);
 		}
 	}
 
 	fn broadcast_consensus_message(&self, message: Bytes) {
-		self.notify(|notify| notify.broadcast(ChainMessageType::Consensus(message.clone())));
+		self.notify(|notify| {
+			notify.broadcast(ChainMessageType::Consensus(message.clone()))
+		});
 	}
 
 	fn epoch_transition_for(&self, parent_hash: H256) -> Option<::engines::EpochTransition> {
@@ -2559,6 +2612,141 @@ impl ProvingBlockChainClient for Client {
 
 impl SnapshotClient for Client {}
 
+impl ImportExportBlocks for Client {
+	fn export_blocks<'a>(
+		&self,
+		mut out: Box<dyn std::io::Write + 'a>,
+		from: BlockId,
+		to: BlockId,
+		format: Option<DataFormat>
+	) -> Result<(), String> {
+		let from = self.block_number(from).ok_or("Starting block could not be found")?;
+		let to = self.block_number(to).ok_or("End block could not be found")?;
+		let format = format.unwrap_or_default();
+
+		for i in from..=to {
+			if i % 10000 == 0 {
+				info!("#{}", i);
+			}
+			let b = self.block(BlockId::Number(i))
+				.ok_or("Error exporting incomplete chain")?
+				.into_inner();
+			match format {
+				DataFormat::Binary => {
+					out.write(&b)
+						.map_err(|e| {
+							format!("Couldn't write to stream. Cause: {}", e)
+						})?;
+				}
+				DataFormat::Hex => {
+					out.write_fmt(format_args!("{}\n", b.pretty()))
+						.map_err(|e| {
+							format!("Couldn't write to stream. Cause: {}", e)
+						})?;
+				}
+			}
+		}
+		Ok(())
+	}
+
+	fn import_blocks<'a>(
+		&self,
+		mut source: Box<dyn std::io::Read + 'a>,
+		format: Option<DataFormat>
+	) -> Result<(), String> {
+		const READAHEAD_BYTES: usize = 8;
+
+		let mut first_bytes: Vec<u8> = vec![0; READAHEAD_BYTES];
+		let mut first_read = 0;
+
+		let format = match format {
+			Some(format) => format,
+			None => {
+				first_read = source.read(&mut first_bytes)
+					.map_err(|_| {
+						"Error reading from the file/stream."
+					})?;
+				match first_bytes[0] {
+					0xf9 => DataFormat::Binary,
+					_ => DataFormat::Hex,
+				}
+			}
+		};
+
+		let do_import = |bytes: Vec<u8>| {
+			let block = Unverified::from_rlp(bytes).map_err(|_| "Invalid block rlp")?;
+			let number = block.header.number();
+			while self.queue_info().is_full() {
+				std::thread::sleep(Duration::from_secs(1));
+			}
+			match self.import_block(block) {
+				Err(EthcoreError::Import(ImportError::AlreadyInChain)) => {
+					trace!("Skipping block #{}: already in chain.", number);
+				}
+				Err(e) => {
+					return Err(format!("Cannot import block #{}: {:?}", number, e));
+				},
+				Ok(_) => {},
+			}
+			Ok(())
+		};
+
+		match format {
+			DataFormat::Binary => {
+				loop {
+					let (mut bytes, n) = if first_read > 0 {
+						(first_bytes.clone(), first_read)
+					} else {
+						let mut bytes = vec![0; READAHEAD_BYTES];
+						let n = source.read(&mut bytes)
+							.map_err(|err| {
+								format!("Error reading from the file/stream: {:?}", err)
+							})?;
+						(bytes, n)
+					};
+					if n == 0 { break; }
+					first_read = 0;
+					let s = PayloadInfo::from(&bytes)
+						.map_err(|e| {
+							format!("Invalid RLP in the file/stream: {:?}", e)
+						})?.total();
+					bytes.resize(s, 0);
+					source.read_exact(&mut bytes[n..])
+						.map_err(|err| {
+							format!("Error reading from the file/stream: {:?}", err)
+						})?;
+					do_import(bytes)?;
+				}
+			}
+			DataFormat::Hex => {
+				for line in BufReader::new(source).lines() {
+					let s = line
+						.map_err(|err| {
+							format!("Error reading from the file/stream: {:?}", err)
+						})?;
+					let s = if first_read > 0 {
+						from_utf8(&first_bytes)
+							.map_err(|err| {
+								format!("Invalid UTF-8: {:?}", err)
+							})?
+							.to_owned() + &(s[..])
+					} else {
+						s
+					};
+					first_read = 0;
+					let bytes = s.from_hex()
+						.map_err(|err| {
+							format!("Invalid hex in file/stream: {:?}", err)
+						})?;
+					do_import(bytes)?;
+				}
+			}
+		};
+		self.flush_queue();
+		Ok(())
+	}
+}
+
 /// Returns `LocalizedReceipt` given `LocalizedTransaction`
 /// and a vector of receipts from given block up to transaction index.
 fn transaction_receipt(
@@ -2606,7 +2794,28 @@ fn transaction_receipt(
 
 #[cfg(test)]
 mod tests {
-	use ethereum_types::{H256, Address};
+	use std::sync::Arc;
+	use std::sync::atomic::{AtomicBool, Ordering};
+	use std::thread;
+	use std::time::Duration;
+
+	use ethereum_types::{Address, H256};
+	use hash::keccak;
+	use kvdb::DBTransaction;
+
+	use blockchain::{ExtrasInsert, BlockProvider};
+	use super::{BlockChainClient, ChainInfo};
+	use ethkey::KeyPair;
+	use types::{
+		encoded,
+		engines::ForkChoice,
+		ids::{BlockId, TransactionId},
+		log_entry::{LocalizedLogEntry, LogEntry},
+		receipt::{LocalizedReceipt, Receipt, TransactionOutcome},
+		transaction::{Action, LocalizedTransaction, Transaction},
+	};
+	use test_helpers::{generate_dummy_client, generate_dummy_client_with_data, generate_dummy_client_with_spec_and_data, get_good_dummy_block_hash};
+	use super::transaction_receipt;
 
 	#[test]
 	fn should_not_cache_details_before_commit() {
@@ -2760,42 +2969,64 @@ mod tests {
 			outcome: TransactionOutcome::StateRoot(state_root),
 		});
 	}
+
+	#[test]
+	fn should_mark_finalization_correctly_for_parent() {
+		let client = generate_dummy_client_with_spec_and_data(::spec::Spec::new_test_with_finality, 2, 0, &[]);
+		let chain = client.chain();
+
+		let block1_details = chain.block_hash(1).and_then(|h| chain.block_details(&h));
+		assert!(block1_details.is_some());
+		let block1_details = block1_details.unwrap();
+		assert_eq!(block1_details.children.len(), 1);
+		assert!(block1_details.is_finalized);
+
+		let block2_details = chain.block_hash(2).and_then(|h| chain.block_details(&h));
+		assert!(block2_details.is_some());
+		let block2_details = block2_details.unwrap();
+		assert_eq!(block2_details.children.len(), 0);
+		assert!(!block2_details.is_finalized);
+	}
 }
 
 /// Queue some items to be processed by IO client.
 struct IoChannelQueue {
-	currently_queued: Arc<AtomicUsize>,
-	limit: usize,
+	/// Using a *signed* integer for counting currently queued messages since the
+	/// order in which the counter is incremented and decremented is not defined.
+	/// Using an unsigned integer can (and will) result in integer underflow,
+	/// incorrectly rejecting messages and returning a FullQueue error.
+	currently_queued: Arc<AtomicI64>,
+	limit: i64,
 }
 
 impl IoChannelQueue {
 	pub fn new(limit: usize) -> Self {
+		let limit = i64::try_from(limit).unwrap_or(i64::max_value());
 		IoChannelQueue {
 			currently_queued: Default::default(),
 			limit,
 		}
 	}
 
-	pub fn queue<F>(&self, channel: &IoChannel<ClientIoMessage>, count: usize, fun: F) -> Result<(), QueueError> where
+	pub fn queue<F>(&self, channel: &IoChannel<ClientIoMessage>, count: usize, fun: F) -> EthcoreResult<()> where
 		F: Fn(&Client) + Send + Sync + 'static,
 	{
 		let queue_size = self.currently_queued.load(AtomicOrdering::Relaxed);
 		if queue_size >= self.limit {
-			return Err(QueueError::Full(self.limit))
+			let err_limit = usize::try_from(self.limit).unwrap_or(usize::max_value());
+			return Err(EthcoreError::Queue(QueueError::Full(err_limit)))
 		};
 
+		let count = i64::try_from(count).unwrap_or(i64::max_value());
+
 		let currently_queued = self.currently_queued.clone();
-		let result = channel.send(ClientIoMessage::execute(move |client| {
+		let _ok = channel.send(ClientIoMessage::execute(move |client| {
 			currently_queued.fetch_sub(count, AtomicOrdering::SeqCst);
 			fun(client);
-		}));
+		}))?;
 
-		match result {
-			Ok(_) => {
-				self.currently_queued.fetch_add(count, AtomicOrdering::SeqCst);
-				Ok(())
-			},
-			Err(e) => return Err(QueueError::Channel(e)),
-		}
+		self.currently_queued.fetch_add(count, AtomicOrdering::SeqCst);
+		Ok(())
 	}
 }
+

ethcore/src/client/config.rs

@@ -117,7 +117,7 @@ pub struct ClientConfig {
 	pub history: u64,
 	/// Ideal memory usage for state pruning history.
 	pub history_mem: usize,
-	/// Check seal valididity on block import
+	/// Check seal validity on block import
 	pub check_seal: bool,
 	/// Maximal number of transactions queued for verification in a separate thread.
 	pub transaction_verification_queue_size: usize,

ethcore/src/client/mod.rs

@@ -33,12 +33,12 @@ pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChain
 pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess};
 pub use self::io_message::ClientIoMessage;
 #[cfg(any(test, feature = "test-helpers"))]
-pub use self::test_client::{TestBlockChainClient, EachBlockWith};
+pub use self::test_client::{TestBlockChainClient, TestState, EachBlockWith};
 pub use self::chain_notify::{ChainNotify, NewBlocks, ChainRoute, ChainRouteType, ChainMessageType};
 pub use self::traits::{
     Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, PrepareOpenBlock, TransactionInfo, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock,
     StateOrBlock, StateClient, Call, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter, BadBlocks,
-	BlockChainReset
+	BlockChainReset, ImportExportBlocks, ForceUpdateSealing
 };
 pub use state::StateInfo;
 pub use self::traits::{BlockChainClient, EngineClient, ProvingBlockChainClient, IoClient};

ethcore/src/client/test_client.rs

@@ -20,6 +20,7 @@ use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrder};
 use std::sync::Arc;
 use std::collections::{HashMap, BTreeMap};
+use blockchain::BlockProvider;
 use std::mem;
 
 use blockchain::{TreeRoute, BlockReceipts};
@@ -57,7 +58,7 @@ use client::{
 	TransactionId, UncleId, TraceId, TraceFilter, LastHashes, CallAnalytics,
 	ProvingBlockChainClient, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock,
 	Call, StateClient, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter, IoClient,
-	BadBlocks
+	BadBlocks, ForceUpdateSealing
 };
 use engines::Engine;
 use error::{Error, EthcoreResult};
@@ -586,7 +587,7 @@ impl ImportBlock for TestBlockChainClient {
 
 impl Call for TestBlockChainClient {
 	// State will not be used by test client anyway, since all methods that accept state are mocked
-	type State = ();
+	type State = TestState;
 
 	fn call(&self, _t: &SignedTransaction, _analytics: CallAnalytics, _state: &mut Self::State, _header: &Header) -> Result<Executed, CallError> {
 		self.execution_result.read().clone().unwrap()
@@ -605,26 +606,30 @@ impl Call for TestBlockChainClient {
 	}
 }
 
-impl StateInfo for () {
+impl StateClient for TestBlockChainClient {
+	// State will not be used by test client anyway, since all methods that accept state are mocked
+	type State = TestState;
+
+	fn latest_state_and_header(&self) -> (Self::State, Header) {
+		(TestState, self.best_block_header())
+	}
+
+	fn state_at(&self, _id: BlockId) -> Option<Self::State> {
+		Some(TestState)
+	}
+}
+
+/// NewType wrapper around `()` to impersonate `State` in trait impls. State will not be used by
+/// test client, since all methods that accept state are mocked.
+pub struct TestState;
+
+impl StateInfo for TestState {
 	fn nonce(&self, _address: &Address) -> ethtrie::Result<U256> { unimplemented!() }
 	fn balance(&self, _address: &Address) -> ethtrie::Result<U256> { unimplemented!() }
 	fn storage_at(&self, _address: &Address, _key: &H256) -> ethtrie::Result<H256> { unimplemented!() }
 	fn code(&self, _address: &Address) -> ethtrie::Result<Option<Arc<Bytes>>> { unimplemented!() }
 }
 
-impl StateClient for TestBlockChainClient {
-	// State will not be used by test client anyway, since all methods that accept state are mocked
-	type State = ();
-
-	fn latest_state(&self) -> Self::State {
-		()
-	}
-
-	fn state_at(&self, _id: BlockId) -> Option<Self::State> {
-		Some(())
-	}
-}
-
 impl EngineInfo for TestBlockChainClient {
 	fn engine(&self) -> &dyn Engine {
 		unimplemented!()
@@ -699,6 +704,10 @@ impl BlockChainClient for TestBlockChainClient {
 		}
 	}
 
+	fn chain(&self) -> Arc<BlockProvider> {
+		unimplemented!()
+	}
+
 	fn list_accounts(&self, _id: BlockId, _after: Option<&Address>, _count: u64) -> Option<Vec<Address>> {
 		None
 	}
@@ -933,8 +942,8 @@ impl ProvingBlockChainClient for TestBlockChainClient {
 }
 
 impl super::traits::EngineClient for TestBlockChainClient {
-	fn update_sealing(&self) {
-		self.miner.update_sealing(self)
+	fn update_sealing(&self, force: ForceUpdateSealing)  {
+		self.miner.update_sealing(self, force)
 	}
 
 	fn submit_seal(&self, block_hash: H256, seal: Vec<Bytes>) {

ethcore/src/client/traits.rs

@@ -17,7 +17,7 @@
 use std::collections::BTreeMap;
 use std::sync::Arc;
 
-use blockchain::{BlockReceipts, TreeRoute};
+use blockchain::{BlockReceipts, TreeRoute, BlockProvider};
 use bytes::Bytes;
 use call_contract::{CallContract, RegistryInfo};
 use ethcore_miner::pool::VerifiedTransaction;
@@ -31,6 +31,7 @@ use types::basic_account::BasicAccount;
 use types::block_status::BlockStatus;
 use types::blockchain_info::BlockChainInfo;
 use types::call_analytics::CallAnalytics;
+use types::data_format::DataFormat;
 use types::encoded;
 use types::filter::Filter;
 use types::header::Header;
@@ -145,7 +146,7 @@ pub trait StateClient {
 	type State: StateInfo;
 
 	/// Get a copy of the best block's state.
-	fn latest_state(&self) -> Self::State;
+	fn latest_state_and_header(&self) -> (Self::State, Header);
 
 	/// Attempt to get a copy of a specific block's final state.
 	///
@@ -237,6 +238,8 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 			.expect("code will return Some if given BlockId::Latest; qed")
 	}
 
+	fn chain(&self) -> Arc<BlockProvider>;
+
 	/// Get block queue information.
 	fn queue_info(&self) -> BlockQueueInfo;
 
@@ -419,10 +422,19 @@ pub trait BroadcastProposalBlock {
 /// Provides methods to import sealed block and broadcast a block proposal
 pub trait SealedBlockImporter: ImportSealedBlock + BroadcastProposalBlock {}
 
+/// Do we want to force update sealing?
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum ForceUpdateSealing {
+	/// Ideally you want to use `No` at all times as `Yes` skips `reseal_required` checks.
+	Yes,
+	/// Don't skip `reseal_required` checks
+	No
+}
+
 /// Client facilities used by internally sealing Engines.
 pub trait EngineClient: Sync + Send + ChainInfo {
 	/// Make a new block and seal it.
-	fn update_sealing(&self);
+	fn update_sealing(&self, force: ForceUpdateSealing);
 
 	/// Submit a seal for a block in the mining queue.
 	fn submit_seal(&self, block_hash: H256, seal: Vec<Bytes>);
@@ -474,3 +486,29 @@ pub trait BlockChainReset {
 	/// reset to best_block - n
 	fn reset(&self, num: u32) -> Result<(), String>;
 }
+
+/// Provides a method for importing/exporting blocks
+pub trait ImportExportBlocks {
+    /// Export blocks to destination, with the given from, to and format argument.
+    /// destination could be a file or stdout.
+    /// If the format is hex, each block is written on a new line.
+    /// For binary exports, all block data is written to the same line.
+	fn export_blocks<'a>(
+        &self,
+        destination: Box<dyn std::io::Write + 'a>,
+        from: BlockId,
+        to: BlockId,
+        format: Option<DataFormat>
+    ) -> Result<(), String>;
+
+	/// Import blocks from destination, with the given format argument
+	/// Source could be a file or stdout.
+	/// For hex format imports, it attempts to read the blocks on a line by line basis.
+	/// For binary format imports, reads the 8 byte RLP header in order to decode the block
+	/// length to be read.
+	fn import_blocks<'a>(
+		&self,
+		source: Box<dyn std::io::Read + 'a>,
+		format: Option<DataFormat>
+	) -> Result<(), String>;
+}