Limit the number of transactions in pending set (#8777)

* Unordered iterator.

* Use unordered and limited set if full not required.

* Split timeout work into smaller timers.

* Avoid collecting all pending transactions when mining

* Remove println.

* Use priority ordering in eth-filter.

* Fix ethcore-miner tests and tx propagation.

* Review grumbles addressed.

* Add test for unordered not populating the cache.

* Fix ethcore tests.

* Fix light tests.

* Fix ethcore-sync tests.

* Fix RPC tests.

transaction-pool/src/pool.rs

@@ -15,7 +15,8 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::sync::Arc;
-use std::collections::{HashMap, BTreeSet};
+use std::slice;
+use std::collections::{hash_map, HashMap, BTreeSet};
 
 use error;
 use listener::{Listener, NoopListener};
@@ -416,7 +417,16 @@ impl<T, S, L> Pool<T, S, L> where
 		PendingIterator {
 			ready,
 			best_transactions,
-			pool: self
+			pool: self,
+		}
+	}
+
+	/// Returns unprioritized list of ready transactions.
+	pub fn unordered_pending<R: Ready<T>>(&self, ready: R) -> UnorderedIterator<T, R, S> {
+		UnorderedIterator {
+			ready,
+			senders: self.transactions.iter(),
+			transactions: None,
 		}
 	}
 
@@ -482,6 +492,50 @@ impl<T, S, L> Pool<T, S, L> where
 	}
 }
 
+/// An iterator over all pending (ready) transactions in unoredered fashion.
+///
+/// NOTE: Current implementation will iterate over all transactions from particular sender
+/// ordered by nonce, but that might change in the future.
+///
+/// NOTE: the transactions are not removed from the queue.
+/// You might remove them later by calling `cull`.
+pub struct UnorderedIterator<'a, T, R, S> where
+	T: VerifiedTransaction + 'a,
+	S: Scoring<T> + 'a,
+{
+	ready: R,
+	senders: hash_map::Iter<'a, T::Sender, Transactions<T, S>>,
+	transactions: Option<slice::Iter<'a, Transaction<T>>>,
+}
+
+impl<'a, T, R, S> Iterator for UnorderedIterator<'a, T, R, S> where
+	T: VerifiedTransaction,
+	R: Ready<T>,
+	S: Scoring<T>,
+{
+	type Item = Arc<T>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		loop {
+			if let Some(transactions) = self.transactions.as_mut() {
+				if let Some(tx) = transactions.next() {
+					match self.ready.is_ready(&tx) {
+						Readiness::Ready => {
+							return Some(tx.transaction.clone());
+						},
+						state => trace!("[{:?}] Ignoring {:?} transaction.", tx.hash(), state),
+					}
+				}
+			}
+
+			// otherwise fallback and try next sender
+			let next_sender = self.senders.next()?;
+			self.transactions = Some(next_sender.1.iter());
+		}
+	}
+}
+
+
 /// An iterator over all pending (ready) transactions.
 /// NOTE: the transactions are not removed from the queue.
 /// You might remove them later by calling `cull`.

transaction-pool/src/tests/mod.rs

@@ -250,6 +250,66 @@ fn should_construct_pending() {
 	assert_eq!(pending.next(), None);
 }
 
+#[test]
+fn should_return_unordered_iterator() {
+	// given
+	let b = TransactionBuilder::default();
+	let mut txq = TestPool::default();
+
+	let tx0 = txq.import(b.tx().nonce(0).gas_price(5).new()).unwrap();
+	let tx1 = txq.import(b.tx().nonce(1).gas_price(5).new()).unwrap();
+	let tx2 = txq.import(b.tx().nonce(2).new()).unwrap();
+	let tx3 = txq.import(b.tx().nonce(3).gas_price(4).new()).unwrap();
+	//gap
+	txq.import(b.tx().nonce(5).new()).unwrap();
+
+	let tx5 = txq.import(b.tx().sender(1).nonce(0).new()).unwrap();
+	let tx6 = txq.import(b.tx().sender(1).nonce(1).new()).unwrap();
+	let tx7 = txq.import(b.tx().sender(1).nonce(2).new()).unwrap();
+	let tx8 = txq.import(b.tx().sender(1).nonce(3).gas_price(4).new()).unwrap();
+	// gap
+	txq.import(b.tx().sender(1).nonce(5).new()).unwrap();
+
+	let tx9 = txq.import(b.tx().sender(2).nonce(0).new()).unwrap();
+	assert_eq!(txq.light_status().transaction_count, 11);
+	assert_eq!(txq.status(NonceReady::default()), Status {
+		stalled: 0,
+		pending: 9,
+		future: 2,
+	});
+	assert_eq!(txq.status(NonceReady::new(1)), Status {
+		stalled: 3,
+		pending: 6,
+		future: 2,
+	});
+
+	// when
+	let all: Vec<_> = txq.unordered_pending(NonceReady::default()).collect();
+
+	let chain1 = vec![tx0, tx1, tx2, tx3];
+	let chain2 = vec![tx5, tx6, tx7, tx8];
+	let chain3 = vec![tx9];
+
+	assert_eq!(all.len(), chain1.len() + chain2.len() + chain3.len());
+
+	let mut options = vec![
+		vec![chain1.clone(), chain2.clone(), chain3.clone()],
+		vec![chain2.clone(), chain1.clone(), chain3.clone()],
+		vec![chain2.clone(), chain3.clone(), chain1.clone()],
+		vec![chain3.clone(), chain2.clone(), chain1.clone()],
+		vec![chain3.clone(), chain1.clone(), chain2.clone()],
+		vec![chain1.clone(), chain3.clone(), chain2.clone()],
+	].into_iter().map(|mut v| {
+		let mut first = v.pop().unwrap();
+		for mut x in v {
+			first.append(&mut x);
+		}
+		first
+	});
+
+	assert!(options.any(|opt| all == opt));
+}
+
 #[test]
 fn should_update_scoring_correctly() {
 	// given