Backports to 1.11.7-stable (#9093)

* parity-version: stabelize 1.11

* parity-version: bump stable to 1.11.7

* Don't fetch snapshot chunks at random (#9088)

* Offload cull to IoWorker.

* Limit the number of transactions in pending set (#8777)

* Unordered iterator.

* Use unordered and limited set if full not required.

* Split timeout work into smaller timers.

* Avoid collecting all pending transactions when mining

* Remove println.

* Use priority ordering in eth-filter.

* Fix ethcore-miner tests and tx propagation.

* Review grumbles addressed.

* Add test for unordered not populating the cache.

* Fix ethcore tests.

* Fix light tests.

* Fix ethcore-sync tests.

* Fix RPC tests.

* Make sure to produce full blocks.

* Update hidapi, fixes #7542 (#9108)

* docker: add cmake dependency (#9111)

* Fix miner tests.

* Revert "Make sure to produce full blocks."

This reverts commit b12d5920b2.

* Update light client hardcoded headers (#9098)

* Insert Kovan hardcoded headers until #7690241

* Insert Kovan hardcoded headers until block 7690241

* Insert Ropsten hardcoded headers until #3612673

* Insert Mainnet hardcoded headers until block 5941249

* Make sure to produce full blocks. (#9115)

* Insert ETC (classic) hardcoded headers until block #6170625 (#9121)

* fix verification in ethcore-sync collect_blocks (#9135)

* `evm bench` fix broken dependencies (#9134)

* `evm bench` use valid dependencies

Benchmarks of the `evm` used stale versions of a couple a crates that
this commit fixes!

* fix warnings

ethcore/sync/src/api.rs

@@ -357,6 +357,10 @@ impl SyncProvider for EthSync {
 	}
 }
 
+const PEERS_TIMER: TimerToken = 0;
+const SYNC_TIMER: TimerToken = 1;
+const TX_TIMER: TimerToken = 2;
+
 struct SyncProtocolHandler {
 	/// Shared blockchain client.
 	chain: Arc<BlockChainClient>,
@@ -371,7 +375,9 @@ struct SyncProtocolHandler {
 impl NetworkProtocolHandler for SyncProtocolHandler {
 	fn initialize(&self, io: &NetworkContext, _host_info: &HostInfo) {
 		if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID {
-			io.register_timer(0, Duration::from_secs(1)).expect("Error registering sync timer");
+			io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer");
+			io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer");
+			io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer");
 		}
 	}
 
@@ -396,12 +402,17 @@ impl NetworkProtocolHandler for SyncProtocolHandler {
 		}
 	}
 
-	fn timeout(&self, io: &NetworkContext, _timer: TimerToken) {
+	fn timeout(&self, io: &NetworkContext, timer: TimerToken) {
 		trace_time!("sync::timeout");
 		let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay);
-		self.sync.write().maintain_peers(&mut io);
-		self.sync.write().maintain_sync(&mut io);
-		self.sync.write().propagate_new_transactions(&mut io);
+		match timer {
+			PEERS_TIMER => self.sync.write().maintain_peers(&mut io),
+			SYNC_TIMER => self.sync.write().maintain_sync(&mut io),
+			TX_TIMER => {
+				self.sync.write().propagate_new_transactions(&mut io);
+			},
+			_ => warn!("Unknown timer {} triggered.", timer),
+		}
 	}
 }
 

ethcore/sync/src/block_sync.rs

@@ -477,18 +477,19 @@ impl BlockDownloader {
 		for block_and_receipts in blocks {
 			let block = block_and_receipts.block;
 			let receipts = block_and_receipts.receipts;
+
+			// Perform basic block verification
+			if !Block::is_good(&block) {
+				debug!(target: "sync", "Bad block rlp: {:?}", block);
+				bad = true;
+				break;
+			}
+
 			let (h, number, parent) = {
 				let header = view!(BlockView, &block).header_view();
 				(header.hash(), header.number(), header.parent_hash())
 			};
 
-			// Perform basic block verification
-			if !Block::is_good(&block) {
-				debug!(target: "sync", "Bad block rlp {:?} : {:?}", h, block);
-				bad = true;
-				break;
-			}
-
 			if self.target_hash.as_ref().map_or(false, |t| t == &h) {
 				self.state = State::Complete;
 				trace!(target: "sync", "Sync target reached");

ethcore/sync/src/chain/mod.rs

@@ -149,6 +149,10 @@ const MAX_NEW_HASHES: usize = 64;
 const MAX_NEW_BLOCK_AGE: BlockNumber = 20;
 // maximal packet size with transactions (cannot be greater than 16MB - protocol limitation).
 const MAX_TRANSACTION_PACKET_SIZE: usize = 8 * 1024 * 1024;
+// Maximal number of transactions queried from miner to propagate.
+// This set is used to diff with transactions known by the peer and
+// we will send a difference of length up to `MAX_TRANSACTIONS_TO_PROPAGATE`.
+const MAX_TRANSACTIONS_TO_QUERY: usize = 4096;
 // Maximal number of transactions in sent in single packet.
 const MAX_TRANSACTIONS_TO_PROPAGATE: usize = 64;
 // Min number of blocks to be behind for a snapshot sync
@@ -1144,7 +1148,7 @@ pub mod tests {
 	use super::{PeerInfo, PeerAsking};
 	use ethcore::header::*;
 	use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient, ChainInfo, BlockInfo};
-	use ethcore::miner::MinerService;
+	use ethcore::miner::{MinerService, PendingOrdering};
 	use private_tx::NoopPrivateTxHandler;
 
 	pub fn get_dummy_block(order: u32, parent_hash: H256) -> Bytes {
@@ -1357,7 +1361,7 @@ pub mod tests {
 			let mut io = TestIo::new(&mut client, &ss, &queue, None);
 			io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks, false);
 			sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[], &[]);
-			assert_eq!(io.chain.miner.ready_transactions(io.chain).len(), 1);
+			assert_eq!(io.chain.miner.ready_transactions(io.chain, 10, PendingOrdering::Priority).len(), 1);
 		}
 		// We need to update nonce status (because we say that the block has been imported)
 		for h in &[good_blocks[0]] {
@@ -1373,7 +1377,7 @@ pub mod tests {
 		}
 
 		// then
-		assert_eq!(client.miner.ready_transactions(&client).len(), 1);
+		assert_eq!(client.miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 1);
 	}
 
 	#[test]

ethcore/sync/src/chain/propagator.rs

@@ -33,6 +33,7 @@ use super::{
 	MAX_PEERS_PROPAGATION,
 	MAX_TRANSACTION_PACKET_SIZE,
 	MAX_TRANSACTIONS_TO_PROPAGATE,
+	MAX_TRANSACTIONS_TO_QUERY,
 	MIN_PEERS_PROPAGATION,
 	CONSENSUS_DATA_PACKET,
 	NEW_BLOCK_HASHES_PACKET,
@@ -114,7 +115,7 @@ impl SyncPropagator {
 			return 0;
 		}
 
-		let transactions = io.chain().ready_transactions();
+		let transactions = io.chain().ready_transactions(MAX_TRANSACTIONS_TO_QUERY);
 		if transactions.is_empty() {
 			return 0;
 		}

ethcore/sync/src/snapshot.rs

@@ -17,7 +17,6 @@
 use ethcore::snapshot::{ManifestData, SnapshotService};
 use ethereum_types::H256;
 use hash::keccak;
-use rand::{thread_rng, Rng};
 
 use std::collections::HashSet;
 use std::iter::FromIterator;
@@ -114,35 +113,32 @@ impl Snapshot {
 		Err(())
 	}
 
-	/// Find a random chunk to download
+	/// Find a chunk to download
 	pub fn needed_chunk(&mut self) -> Option<H256> {
-		// Find all random chunks: first blocks, then state
-		let needed_chunks = {
+		// Find next needed chunk: first block, then state chunks
+		let chunk = {
 			let chunk_filter = |h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h);
 
-			let needed_block_chunks = self.pending_block_chunks.iter()
+			let needed_block_chunk = self.pending_block_chunks.iter()
 				.filter(|&h| chunk_filter(h))
 				.map(|h| *h)
-				.collect::<Vec<H256>>();
+				.next();
 
 			// If no block chunks to download, get the state chunks
-			if needed_block_chunks.len() == 0 {
+			if needed_block_chunk.is_none() {
 				self.pending_state_chunks.iter()
 					.filter(|&h| chunk_filter(h))
 					.map(|h| *h)
-					.collect::<Vec<H256>>()
+					.next()
 			} else {
-				needed_block_chunks
+				needed_block_chunk
 			}
 		};
 
-		// Get a random chunk
-		let chunk = thread_rng().choose(&needed_chunks);
-
 		if let Some(hash) = chunk {
 			self.downloading_chunks.insert(hash.clone());
 		}
-		chunk.map(|h| *h)
+		chunk
 	}
 
 	pub fn clear_chunk_download(&mut self, hash: &H256) {