A last bunch of txqueue performance optimizations (#9024)

* Clear cache only when block is enacted.

* Add tracing for cull.

* Cull split.

* Cull after creating pending block.

* Add constant, remove sync::read tracing.

* Reset debug.

* Remove excessive tracing.

* Use struct for NonceCache.

* Fix build

* Remove warnings.

* Fix build again.

miner/src/pool/queue.rs

@@ -43,6 +43,14 @@ type Pool = txpool::Pool<pool::VerifiedTransaction, scoring::NonceAndGasPrice, L
 /// since it only affects transaction Condition.
 const TIMESTAMP_CACHE: u64 = 1000;
 
+/// How many senders at once do we attempt to process while culling.
+///
+/// When running with huge transaction pools, culling can take significant amount of time.
+/// To prevent holding `write()` lock on the pool for this long period, we split the work into
+/// chunks and allow other threads to utilize the pool in the meantime.
+/// This parameter controls how many (best) senders at once will be processed.
+const CULL_SENDERS_CHUNK: usize = 1024;
+
 /// Transaction queue status.
 #[derive(Debug, Clone, PartialEq)]
 pub struct Status {
@@ -398,10 +406,11 @@ impl TransactionQueue {
 	}
 
 	/// Culls all stalled transactions from the pool.
-	pub fn cull<C: client::NonceClient>(
+	pub fn cull<C: client::NonceClient + Clone>(
 		&self,
 		client: C,
 	) {
+		trace_time!("pool::cull");
 		// We don't care about future transactions, so nonce_cap is not important.
 		let nonce_cap = None;
 		// We want to clear stale transactions from the queue as well.
@@ -416,10 +425,19 @@ impl TransactionQueue {
 			current_id.checked_sub(gap)
 		};
 
-		let state_readiness = ready::State::new(client, stale_id, nonce_cap);
-
 		self.recently_rejected.clear();
-		let removed = self.pool.write().cull(None, state_readiness);
+
+		let mut removed = 0;
+		let senders: Vec<_> = {
+			let pool = self.pool.read();
+			let senders = pool.senders().cloned().collect();
+			senders
+		};
+		for chunk in senders.chunks(CULL_SENDERS_CHUNK) {
+			trace_time!("pool::cull::chunk");
+			let state_readiness = ready::State::new(client.clone(), stale_id, nonce_cap);
+			removed += self.pool.write().cull(Some(chunk), state_readiness);
+		}
 		debug!(target: "txqueue", "Removed {} stalled transactions. {}", removed, self.status());
 	}