New Transaction Queue implementation (#8074)

* Implementation of Verifier, Scoring and Ready. * Queue in progress. * TransactionPool. * Prepare for txpool release. * Miner refactor [WiP] * WiP reworking miner. * Make it compile. * Add some docs. * Split blockchain access to a separate file. * Work on miner API. * Fix ethcore tests. * Refactor miner interface for sealing/work packages. * Implement next nonce. * RPC compiles. * Implement couple of missing methdods for RPC. * Add transaction queue listeners. * Compiles! * Clean-up and parallelize. * Get rid of RefCell in header. * Revert "Get rid of RefCell in header." This reverts commit 0f2424c9b7319a786e1565ea2a8a6d801a21b4fb. * Override Sync requirement. * Fix status display. * Unify logging. * Extract some cheap checks. * Measurements and optimizations. * Fix scoring bug, heap size of bug and add cache * Disable tx queueing and parallel verification. * Make ethcore and ethcore-miner compile again. * Make RPC compile again. * Bunch of txpool tests. * Migrate transaction queue tests. * Nonce Cap * Nonce cap cache and tests. * Remove stale future transactions from the queue. * Optimize scoring and write some tests. * Simple penalization. * Clean up and support for different scoring algorithms. * Add CLI parameters for the new queue. * Remove banning queue. * Disable debug build. * Change per_sender limit to be 1% instead of 5% * Avoid cloning when propagating transactions. * Remove old todo. * Post-review fixes. * Fix miner options default. * Implement back ready transactions for light client. * Get rid of from_pending_block * Pass rejection reason. * Add more details to drop. * Rollback heap size of. * Avoid cloning hashes when propagating and include more details on rejection. * Fix tests. * Introduce nonces cache. * Remove uneccessary hashes allocation. * Lower the mem limit. * Re-enable parallel verification. * Add miner log. Don't check the type if not below min_gas_price. * Add more traces, fix disabling miner. * Fix creating pending blocks twice on AuRa authorities. * Fix tests. * re-use pending blocks in AuRa * Use reseal_min_period to prevent too frequent update_sealing. * Fix log to contain hash not sender. * Optimize local transactions. * Fix aura tests. * Update locks comments. * Get rid of unsafe Sync impl. * Review fixes. * Remove excessive matches. * Fix compilation errors. * Use new pool in private transactions. * Fix private-tx test. * Fix secret store tests. * Actually use gas_floor_target * Fix config tests. * Fix pool tests. * Address grumbles.
2018-04-13 17:34:27 +02:00
parent 03b96a7c0a
commit 1cd93e4ceb
105 changed files with 5185 additions and 5784 deletions
--- a/ethcore/src/client/chain_notify.rs
+++ b/ethcore/src/client/chain_notify.rs
@@ -14,8 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.

-use ethereum_types::H256;
 use bytes::Bytes;
+use ethereum_types::H256;
+use transaction::UnverifiedTransaction;

 /// Messages to broadcast via chain
 pub enum ChainMessageType {
@@ -59,7 +60,7 @@ pub trait ChainNotify : Send + Sync {

 	/// fires when new transactions are received from a peer
 	fn transactions_received(&self,
-		_hashes: Vec<H256>,
+		_txs: &[UnverifiedTransaction],
 		_peer_id: usize,
 	) {
 		// does nothing by default
--- a/ethcore/src/client/client.rs
+++ b/ethcore/src/client/client.rs
@@ -44,7 +44,7 @@ use client::{
 };
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
-	MiningBlockChainClient, TraceFilter, CallAnalytics, BlockImportError, Mode,
+	TraceFilter, CallAnalytics, BlockImportError, Mode,
 	ChainNotify, PruningInfo, ProvingBlockChainClient, EngineInfo, ChainMessageType
 };
 use encoded;
@@ -58,6 +58,7 @@ use header::{BlockNumber, Header};
 use io::IoChannel;
 use log_entry::LocalizedLogEntry;
 use miner::{Miner, MinerService};
+use ethcore_miner::pool::VerifiedTransaction;
 use parking_lot::{Mutex, RwLock};
 use rand::OsRng;
 use receipt::{Receipt, LocalizedReceipt};
@@ -68,7 +69,7 @@ use state_db::StateDB;
 use state::{self, State};
 use trace;
 use trace::{TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase};
-use transaction::{self, LocalizedTransaction, UnverifiedTransaction, SignedTransaction, Transaction, PendingTransaction, Action};
+use transaction::{self, LocalizedTransaction, UnverifiedTransaction, SignedTransaction, Transaction, Action};
 use types::filter::Filter;
 use types::mode::Mode as IpcMode;
 use verification;
@@ -103,10 +104,10 @@ pub struct ClientReport {

 impl ClientReport {
 	/// Alter internal reporting to reflect the additional `block` has been processed.
-	pub fn accrue_block(&mut self, block: &PreverifiedBlock) {
+	pub fn accrue_block(&mut self, header: &Header, transactions: usize) {
 		self.blocks_imported += 1;
-		self.transactions_applied += block.transactions.len();
-		self.gas_processed = self.gas_processed + block.header.gas_used().clone();
+		self.transactions_applied += transactions;
+		self.gas_processed = self.gas_processed + *header.gas_used();
 	}
 }

@@ -295,23 +296,29 @@ impl Importer {
 			let start = Instant::now();

 			for block in blocks {
-				let header = &block.header;
+				let header = block.header.clone();
+				let bytes = block.bytes.clone();
+				let hash = header.hash();
+
 				let is_invalid = invalid_blocks.contains(header.parent_hash());
 				if is_invalid {
-					invalid_blocks.insert(header.hash());
+					invalid_blocks.insert(hash);
 					continue;
 				}
-				if let Ok(closed_block) = self.check_and_close_block(&block, client) {
-					if self.engine.is_proposal(&block.header) {
-						self.block_queue.mark_as_good(&[header.hash()]);
-						proposed_blocks.push(block.bytes);
-					} else {
-						imported_blocks.push(header.hash());

-						let route = self.commit_block(closed_block, &header, &block.bytes, client);
+				if let Ok(closed_block) = self.check_and_close_block(block, client) {
+					if self.engine.is_proposal(&header) {
+						self.block_queue.mark_as_good(&[hash]);
+						proposed_blocks.push(bytes);
+					} else {
+						imported_blocks.push(hash);
+
+						let transactions_len = closed_block.transactions().len();
+
+						let route = self.commit_block(closed_block, &header, &bytes, client);
 						import_results.push(route);

-						client.report.write().accrue_block(&block);
+						client.report.write().accrue_block(&header, transactions_len);
 					}
 				} else {
 					invalid_blocks.insert(header.hash());
@@ -337,7 +344,7 @@ impl Importer {
 				let (enacted, retracted) = self.calculate_enacted_retracted(&import_results);

 				if is_empty {
-					self.miner.chain_new_blocks(client, &imported_blocks, &invalid_blocks, &enacted, &retracted);
+					self.miner.chain_new_blocks(client, &imported_blocks, &invalid_blocks, &enacted, &retracted, false);
 				}

 				client.notify(|notify| {
@@ -358,9 +365,9 @@ impl Importer {
 		imported
 	}

-	fn check_and_close_block(&self, block: &PreverifiedBlock, client: &Client) -> Result<LockedBlock, ()> {
+	fn check_and_close_block(&self, block: PreverifiedBlock, client: &Client) -> Result<LockedBlock, ()> {
 		let engine = &*self.engine;
-		let header = &block.header;
+		let header = block.header.clone();

 		// Check the block isn't so old we won't be able to enact it.
 		let best_block_number = client.chain.read().best_block_number();
@@ -381,7 +388,7 @@ impl Importer {
 		let chain = client.chain.read();
 		// Verify Block Family
 		let verify_family_result = self.verifier.verify_block_family(
-			header,
+			&header,
 			&parent,
 			engine,
 			Some(verification::FullFamilyParams {
@@ -397,7 +404,7 @@ impl Importer {
 			return Err(());
 		};

-		let verify_external_result = self.verifier.verify_block_external(header, engine);
+		let verify_external_result = self.verifier.verify_block_external(&header, engine);
 		if let Err(e) = verify_external_result {
 			warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			return Err(());
@@ -409,7 +416,8 @@ impl Importer {

 		let is_epoch_begin = chain.epoch_transition(parent.number(), *header.parent_hash()).is_some();
 		let strip_receipts = header.number() < engine.params().validate_receipts_transition;
-		let enact_result = enact_verified(block,
+		let enact_result = enact_verified(
+			block,
 			engine,
 			client.tracedb.read().tracing_enabled(),
 			db,
@@ -425,7 +433,7 @@ impl Importer {
 		})?;

 		// Final Verification
-		if let Err(e) = self.verifier.verify_block_final(header, locked_block.block().header()) {
+		if let Err(e) = self.verifier.verify_block_final(&header, locked_block.block().header()) {
 			warn!(target: "client", "Stage 5 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			return Err(());
 		}
@@ -975,19 +983,24 @@ impl Client {

 	/// Import transactions from the IO queue
 	pub fn import_queued_transactions(&self, transactions: &[Bytes], peer_id: usize) -> usize {
-		trace!(target: "external_tx", "Importing queued");
 		trace_time!("import_queued_transactions");
 		self.queue_transactions.fetch_sub(transactions.len(), AtomicOrdering::SeqCst);
-		let txs: Vec<UnverifiedTransaction> = transactions.iter().filter_map(|bytes| UntrustedRlp::new(bytes).as_val().ok()).collect();
-		let hashes: Vec<_> = txs.iter().map(|tx| tx.hash()).collect();
+
+		let txs: Vec<UnverifiedTransaction> = transactions
+			.iter()
+			.filter_map(|bytes| UntrustedRlp::new(bytes).as_val().ok())
+			.collect();
+
 		self.notify(|notify| {
-			notify.transactions_received(hashes.clone(), peer_id);
+			notify.transactions_received(&txs, peer_id);
 		});
+
 		let results = self.importer.miner.import_external_transactions(self, txs);
 		results.len()
 	}

 	/// Get shared miner reference.
+	#[cfg(test)]
 	pub fn miner(&self) -> Arc<Miner> {
 		self.importer.miner.clone()
 	}
@@ -1915,12 +1928,8 @@ impl BlockChainClient for Client {
 		}
 	}

-	fn ready_transactions(&self) -> Vec<PendingTransaction> {
-		let (number, timestamp) = {
-			let chain = self.chain.read();
-			(chain.best_block_number(), chain.best_block_timestamp())
-		};
-		self.importer.miner.ready_transactions(number, timestamp)
+	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>> {
+		self.importer.miner.ready_transactions(self)
 	}

 	fn queue_consensus_message(&self, message: Bytes) {
@@ -1951,17 +1960,19 @@ impl BlockChainClient for Client {
 		}
 	}

-	fn transact_contract(&self, address: Address, data: Bytes) -> Result<transaction::ImportResult, EthcoreError> {
+	fn transact_contract(&self, address: Address, data: Bytes) -> Result<(), transaction::Error> {
+		let authoring_params = self.importer.miner.authoring_params();
 		let transaction = Transaction {
-			nonce: self.latest_nonce(&self.importer.miner.author()),
+			nonce: self.latest_nonce(&authoring_params.author),
 			action: Action::Call(address),
-			gas: self.importer.miner.gas_floor_target(),
+			gas: self.importer.miner.sensible_gas_limit(),
 			gas_price: self.importer.miner.sensible_gas_price(),
 			value: U256::zero(),
 			data: data,
 		};
 		let chain_id = self.engine.signing_chain_id(&self.latest_env_info());
-		let signature = self.engine.sign(transaction.hash(chain_id))?;
+		let signature = self.engine.sign(transaction.hash(chain_id))
+			.map_err(|e| transaction::Error::InvalidSignature(e.to_string()))?;
 		let signed = SignedTransaction::new(transaction.with_signature(signature, chain_id))?;
 		self.importer.miner.import_own_transaction(self, signed.into())
 	}
@@ -2070,7 +2081,7 @@ impl ImportSealedBlock for Client {
 			route
 		};
 		let (enacted, retracted) = self.importer.calculate_enacted_retracted(&[route]);
-		self.importer.miner.chain_new_blocks(self, &[h.clone()], &[], &enacted, &retracted);
+		self.importer.miner.chain_new_blocks(self, &[h.clone()], &[], &enacted, &retracted, true);
 		self.notify(|notify| {
 			notify.new_blocks(
 				vec![h.clone()],
@@ -2108,11 +2119,8 @@ impl BroadcastProposalBlock for Client {

 impl SealedBlockImporter for Client {}

-impl MiningBlockChainClient for Client {
-	fn vm_factory(&self) -> &VmFactory {
-		&self.factories.vm
-	}
-}
+impl ::miner::TransactionVerifierClient for Client {}
+impl ::miner::BlockChainClient for Client {}

 impl super::traits::EngineClient for Client {
 	fn update_sealing(&self) {
@@ -2120,8 +2128,9 @@ impl super::traits::EngineClient for Client {
 	}

 	fn submit_seal(&self, block_hash: H256, seal: Vec<Bytes>) {
-		if self.importer.miner.submit_seal(self, block_hash, seal).is_err() {
-			warn!(target: "poa", "Wrong internal seal submission!")
+		let import = self.importer.miner.submit_seal(block_hash, seal).and_then(|block| self.import_sealed_block(block));
+		if let Err(err) = import {
+			warn!(target: "poa", "Wrong internal seal submission! {:?}", err);
 		}
 	}

--- a/ethcore/src/client/mod.rs
+++ b/ethcore/src/client/mod.rs
@@ -38,7 +38,7 @@ pub use self::traits::{
 };
 //pub use self::private_notify::PrivateNotify;
 pub use state::StateInfo;
-pub use self::traits::{BlockChainClient, MiningBlockChainClient, EngineClient, ProvingBlockChainClient};
+pub use self::traits::{BlockChainClient, EngineClient, ProvingBlockChainClient};

 pub use types::ids::*;
 pub use types::trace_filter::Filter as TraceFilter;
--- a/ethcore/src/client/test_client.rs
+++ b/ethcore/src/client/test_client.rs
@@ -31,11 +31,12 @@ use kvdb_memorydb;
 use bytes::Bytes;
 use rlp::{UntrustedRlp, RlpStream};
 use ethkey::{Generator, Random};
-use transaction::{self, Transaction, LocalizedTransaction, PendingTransaction, SignedTransaction, Action};
+use ethcore_miner::pool::VerifiedTransaction;
+use transaction::{self, Transaction, LocalizedTransaction, SignedTransaction, Action};
 use blockchain::{TreeRoute, BlockReceipts};
 use client::{
 	Nonce, Balance, ChainInfo, BlockInfo, ReopenBlock, CallContract, TransactionInfo, RegistryInfo,
-	PrepareOpenBlock, BlockChainClient, MiningBlockChainClient, BlockChainInfo, BlockStatus, BlockId,
+	PrepareOpenBlock, BlockChainClient, BlockChainInfo, BlockStatus, BlockId,
 	TransactionId, UncleId, TraceId, TraceFilter, LastHashes, CallAnalytics, BlockImportError,
 	ProvingBlockChainClient, ScheduleInfo, ImportSealedBlock, BroadcastProposalBlock, ImportBlock, StateOrBlock,
 	Call, StateClient, EngineInfo, AccountData, BlockChain, BlockProducer, SealedBlockImporter
@@ -45,9 +46,7 @@ use header::{Header as BlockHeader, BlockNumber};
 use filter::Filter;
 use log_entry::LocalizedLogEntry;
 use receipt::{Receipt, LocalizedReceipt, TransactionOutcome};
-use error::{ImportResult, Error as EthcoreError};
-use evm::VMType;
-use factory::VmFactory;
+use error::ImportResult;
 use vm::Schedule;
 use miner::{Miner, MinerService};
 use spec::Spec;
@@ -102,8 +101,6 @@ pub struct TestBlockChainClient {
 	pub miner: Arc<Miner>,
 	/// Spec
 	pub spec: Spec,
-	/// VM Factory
-	pub vm_factory: VmFactory,
 	/// Timestamp assigned to latest sealed block
 	pub latest_block_timestamp: RwLock<u64>,
 	/// Ancient block info.
@@ -174,9 +171,8 @@ impl TestBlockChainClient {
 			receipts: RwLock::new(HashMap::new()),
 			logs: RwLock::new(Vec::new()),
 			queue_size: AtomicUsize::new(0),
-			miner: Arc::new(Miner::with_spec(&spec)),
+			miner: Arc::new(Miner::new_for_tests(&spec, None)),
 			spec: spec,
-			vm_factory: VmFactory::new(VMType::Interpreter, 1024 * 1024),
 			latest_block_timestamp: RwLock::new(10_000_000),
 			ancient_block: RwLock::new(None),
 			first_block: RwLock::new(None),
@@ -345,8 +341,8 @@ impl TestBlockChainClient {
 		self.set_balance(signed_tx.sender(), 10_000_000_000_000_000_000u64.into());
 		let hash = signed_tx.hash();
 		let res = self.miner.import_external_transactions(self, vec![signed_tx.into()]);
-		let res = res.into_iter().next().unwrap().expect("Successful import");
-		assert_eq!(res, transaction::ImportResult::Current);
+		let res = res.into_iter().next().unwrap();
+		assert!(res.is_ok());
 		hash
 	}

@@ -423,11 +419,8 @@ impl BroadcastProposalBlock for TestBlockChainClient {

 impl SealedBlockImporter for TestBlockChainClient {}

-impl MiningBlockChainClient for TestBlockChainClient {
-	fn vm_factory(&self) -> &VmFactory {
-		&self.vm_factory
-	}
-}
+impl ::miner::TransactionVerifierClient for TestBlockChainClient {}
+impl ::miner::BlockChainClient for TestBlockChainClient {}

 impl Nonce for TestBlockChainClient {
 	fn nonce(&self, address: &Address, id: BlockId) -> Option<U256> {
@@ -826,9 +819,8 @@ impl BlockChainClient for TestBlockChainClient {
 		self.spec.engine.handle_message(&message).unwrap();
 	}

-	fn ready_transactions(&self) -> Vec<PendingTransaction> {
-		let info = self.chain_info();
-		self.miner.ready_transactions(info.best_block_number, info.best_block_timestamp)
+	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>> {
+		self.miner.ready_transactions(self)
 	}

 	fn signing_chain_id(&self) -> Option<u64> { None }
@@ -851,9 +843,9 @@ impl BlockChainClient for TestBlockChainClient {
 		}
 	}

-	fn transact_contract(&self, address: Address, data: Bytes) -> Result<transaction::ImportResult, EthcoreError> {
+	fn transact_contract(&self, address: Address, data: Bytes) -> Result<(), transaction::Error> {
 		let transaction = Transaction {
-			nonce: self.latest_nonce(&self.miner.author()),
+			nonce: self.latest_nonce(&self.miner.authoring_params().author),
 			action: Action::Call(address),
 			gas: self.spec.gas_limit,
 			gas_price: U256::zero(),
@@ -895,8 +887,9 @@ impl super::traits::EngineClient for TestBlockChainClient {
 	}

 	fn submit_seal(&self, block_hash: H256, seal: Vec<Bytes>) {
-		if self.miner.submit_seal(self, block_hash, seal).is_err() {
-			warn!(target: "poa", "Wrong internal seal submission!")
+		let import = self.miner.submit_seal(block_hash, seal).and_then(|block| self.import_sealed_block(block));
+		if let Err(err) = import {
+			warn!(target: "poa", "Wrong internal seal submission! {:?}", err);
 		}
 	}

--- a/ethcore/src/client/traits.rs
+++ b/ethcore/src/client/traits.rs
@@ -15,28 +15,30 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.

 use std::collections::BTreeMap;
+use std::sync::Arc;
+
 use itertools::Itertools;

 use block::{OpenBlock, SealedBlock, ClosedBlock};
 use blockchain::TreeRoute;
 use encoded;
 use vm::LastHashes;
-use error::{ImportResult, CallError, Error as EthcoreError, BlockImportError};
+use error::{ImportResult, CallError, BlockImportError};
 use evm::Schedule;
-use factory::VmFactory;
 use executive::Executed;
 use filter::Filter;
 use header::{BlockNumber};
 use log_entry::LocalizedLogEntry;
 use receipt::LocalizedReceipt;
 use trace::LocalizedTrace;
-use transaction::{LocalizedTransaction, PendingTransaction, SignedTransaction, ImportResult as TransactionImportResult};
+use transaction::{self, LocalizedTransaction, SignedTransaction};
 use verification::queue::QueueInfo as BlockQueueInfo;
 use state::StateInfo;
 use header::Header;
 use engines::EthEngine;

 use ethereum_types::{H256, U256, Address};
+use ethcore_miner::pool::VerifiedTransaction;
 use bytes::Bytes;
 use hashdb::DBValue;

@@ -315,7 +317,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 	fn queue_consensus_message(&self, message: Bytes);

 	/// List all transactions that are allowed into the next block.
-	fn ready_transactions(&self) -> Vec<PendingTransaction>;
+	fn ready_transactions(&self) -> Vec<Arc<VerifiedTransaction>>;

 	/// Sorted list of transaction gas prices from at least last sample_size blocks.
 	fn gas_price_corpus(&self, sample_size: usize) -> ::stats::Corpus<U256> {
@@ -366,8 +368,8 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 	/// Returns information about pruning/data availability.
 	fn pruning_info(&self) -> PruningInfo;

-	/// Import a transaction: used for misbehaviour reporting.
-	fn transact_contract(&self, address: Address, data: Bytes) -> Result<TransactionImportResult, EthcoreError>;
+	/// Schedule state-altering transaction to be executed on the next pending block.
+	fn transact_contract(&self, address: Address, data: Bytes) -> Result<(), transaction::Error>;

 	/// Get the address of the registry itself.
 	fn registrar_address(&self) -> Option<Address>;
@@ -416,12 +418,6 @@ pub trait BroadcastProposalBlock {
 /// Provides methods to import sealed block and broadcast a block proposal
 pub trait SealedBlockImporter: ImportSealedBlock + BroadcastProposalBlock {}

-/// Extended client interface used for mining
-pub trait MiningBlockChainClient: BlockChainClient + BlockProducer + ScheduleInfo + SealedBlockImporter {
-	/// Returns EvmFactory.
-	fn vm_factory(&self) -> &VmFactory;
-}
-
 /// Client facilities used by internally sealing Engines.
 pub trait EngineClient: Sync + Send + ChainInfo {
 	/// Make a new block and seal it.