Limit the number of transactions in pending set (#8777)

* Unordered iterator.

* Use unordered and limited set if full not required.

* Split timeout work into smaller timers.

* Avoid collecting all pending transactions when mining

* Remove println.

* Use priority ordering in eth-filter.

* Fix ethcore-miner tests and tx propagation.

* Review grumbles addressed.

* Add test for unordered not populating the cache.

* Fix ethcore tests.

* Fix light tests.

* Fix ethcore-sync tests.

* Fix RPC tests.

rpc/src/v1/impls/eth_filter.rs

@@ -85,7 +85,7 @@ impl<C, M> Filterable for EthFilterClient<C, M> where
 	}
 
 	fn pending_transactions_hashes(&self) -> Vec<H256> {
-		self.miner.ready_transactions(&*self.client)
+		self.miner.ready_transactions(&*self.client, usize::max_value(), miner::PendingOrdering::Priority)
 			.into_iter()
 			.map(|tx| tx.signed().hash())
 			.collect()

rpc/src/v1/impls/eth_pubsub.rs

@@ -175,7 +175,7 @@ impl<C> ChainNotificationHandler<C> {
 	}
 
 	/// Notify all subscribers about new transaction hashes.
-	pub fn new_transactions(&self, hashes: &[H256]) {
+	pub fn notify_new_transactions(&self, hashes: &[H256]) {
 		for subscriber in self.transactions_subscribers.read().values() {
 			for hash in hashes {
 				Self::notify(&self.remote, subscriber, pubsub::Result::TransactionHash((*hash).into()));

rpc/src/v1/impls/light/parity.rs

@@ -264,12 +264,13 @@ impl Parity for ParityClient {
 			.map(Into::into)
 	}
 
-	fn pending_transactions(&self) -> Result<Vec<Transaction>> {
+	fn pending_transactions(&self, limit: Trailing<usize>) -> Result<Vec<Transaction>> {
 		let txq = self.light_dispatch.transaction_queue.read();
 		let chain_info = self.light_dispatch.client.chain_info();
 		Ok(
 			txq.ready_transactions(chain_info.best_block_number, chain_info.best_block_timestamp)
 				.into_iter()
+				.take(limit.unwrap_or_else(usize::max_value))
 				.map(|tx| Transaction::from_pending(tx, chain_info.best_block_number, self.eip86_transition))
 				.collect::<Vec<_>>()
 		)

rpc/src/v1/impls/parity.rs

@@ -303,15 +303,19 @@ impl<C, M, U, S> Parity for ParityClient<C, M, U> where
 			.map(Into::into)
 	}
 
-	fn pending_transactions(&self) -> Result<Vec<Transaction>> {
+	fn pending_transactions(&self, limit: Trailing<usize>) -> Result<Vec<Transaction>> {
 		let block_number = self.client.chain_info().best_block_number;
-		let ready_transactions = self.miner.ready_transactions(&*self.client);
+		let ready_transactions = self.miner.ready_transactions(
+			&*self.client,
+			limit.unwrap_or_else(usize::max_value),
+			miner::PendingOrdering::Priority,
+		);
 
 		Ok(ready_transactions
 		   .into_iter()
 		   .map(|t| Transaction::from_pending(t.pending().clone(), block_number, self.eip86_transition))
 		   .collect()
-	  )
+		)
 	}
 
 	fn all_transactions(&self) -> Result<Vec<Transaction>> {

rpc/src/v1/tests/helpers/miner_service.rs

@@ -27,7 +27,7 @@ use ethcore::engines::EthEngine;
 use ethcore::error::Error;
 use ethcore::header::{BlockNumber, Header};
 use ethcore::ids::BlockId;
-use ethcore::miner::{MinerService, AuthoringParams};
+use ethcore::miner::{self, MinerService, AuthoringParams};
 use ethcore::receipt::{Receipt, RichReceipt};
 use ethereum_types::{H256, U256, Address};
 use miner::pool::local_transactions::Status as LocalTransactionStatus;
@@ -208,7 +208,7 @@ impl MinerService for TestMinerService {
 		self.local_transactions.lock().iter().map(|(hash, stats)| (*hash, stats.clone())).collect()
 	}
 
-	fn ready_transactions<C>(&self, _chain: &C) -> Vec<Arc<VerifiedTransaction>> {
+	fn ready_transactions<C>(&self, _chain: &C, _max_len: usize, _ordering: miner::PendingOrdering) -> Vec<Arc<VerifiedTransaction>> {
 		self.queued_transactions()
 	}
 

rpc/src/v1/tests/mocked/eth_pubsub.rs

@@ -181,7 +181,7 @@ fn should_subscribe_to_pending_transactions() {
 	assert_eq!(io.handle_request_sync(request, metadata.clone()), Some(response.to_owned()));
 
 	// Send new transactions
-	handler.new_transactions(&[5.into(), 7.into()]);
+	handler.notify_new_transactions(&[5.into(), 7.into()]);
 
 	let (res, receiver) = receiver.into_future().wait().unwrap();
 	let response = r#"{"jsonrpc":"2.0","method":"eth_subscription","params":{"result":"0x0000000000000000000000000000000000000000000000000000000000000005","subscription":"0x416d77337e24399d"}}"#;

rpc/src/v1/traits/parity.rs

@@ -141,7 +141,7 @@ build_rpc_trait! {
 
 		/// Returns all pending transactions from transaction queue.
 		#[rpc(name = "parity_pendingTransactions")]
-		fn pending_transactions(&self) -> Result<Vec<Transaction>>;
+		fn pending_transactions(&self, Trailing<usize>) -> Result<Vec<Transaction>>;
 
 		/// Returns all transactions from transaction queue.
 		///