Generalize engine trait (#6591)

* move common forks and parameters to common params

* port specs over to new format

* fix RPC tests

* parity-machine skeleton

* remove block type

* extract out ethereum-specific methods into EthereumMachine

* beginning to integrate Machine into engines. dealing with stale transitions in Ethash

* initial porting to machine

* move block reward back into engine

* abstract block reward logic

* move last hash and DAO HF logic into machine

* begin making engine function parameters generic

* abstract epoch verifier and ethash block reward logic

* instantiate special ethereummachine for ethash in spec

* optional full verification in verify_block_family

* re-instate tx_filter in a way that works for all engines

* fix warnings

* fix most tests, further generalize engine trait

* uncomment nullengine, get ethcore tests compiling

* fix warnings

* update a bunch of specs

* re-enable engine signer, validator set, and transition handler

* migrate basic_authority engine

* move last hashes into executedblock

* port tendermint

* make all ethcore tests pass

* json-tests compilation

* fix RPC tests: change in gas limit for new block changed PoW hash

* fix minor grumbles

* validate chainspecs

* fix broken import

* fix transaction verification for pre-homestead

ethcore/src/verification/canon_verifier.rs

@@ -16,8 +16,7 @@
 
 //! Canonical verifier.
 
-use blockchain::BlockProvider;
-use engines::Engine;
+use engines::EthEngine;
 use error::Error;
 use header::Header;
 use super::Verifier;
@@ -27,15 +26,21 @@ use super::verification;
 pub struct CanonVerifier;
 
 impl Verifier for CanonVerifier {
-	fn verify_block_family(&self, header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> {
-		verification::verify_block_family(header, bytes, engine, bc)
+	fn verify_block_family(
+		&self,
+		header: &Header,
+		parent: &Header,
+		engine: &EthEngine,
+		do_full: Option<verification::FullFamilyParams>,
+	) -> Result<(), Error> {
+		verification::verify_block_family(header, parent, engine, do_full)
 	}
 
 	fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error> {
 		verification::verify_block_final(expected, got)
 	}
 
-	fn verify_block_external(&self, header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> {
-		engine.verify_block_external(header, Some(bytes))
+	fn verify_block_external(&self, header: &Header, engine: &EthEngine) -> Result<(), Error> {
+		engine.verify_block_external(header)
 	}
 }

ethcore/src/verification/noop_verifier.rs

@@ -16,18 +16,23 @@
 
 //! No-op verifier.
 
-use blockchain::BlockProvider;
-use engines::Engine;
+use engines::EthEngine;
 use error::Error;
 use header::Header;
-use super::Verifier;
+use super::{verification, Verifier};
 
 /// A no-op verifier -- this will verify everything it's given immediately.
 #[allow(dead_code)]
 pub struct NoopVerifier;
 
 impl Verifier for NoopVerifier {
-	fn verify_block_family(&self, _header: &Header, _bytes: &[u8], _engine: &Engine, _bc: &BlockProvider) -> Result<(), Error> {
+	fn verify_block_family(
+		&self,
+		_: &Header,
+		_t: &Header,
+		_: &EthEngine,
+		_: Option<verification::FullFamilyParams>
+	) -> Result<(), Error> {
 		Ok(())
 	}
 
@@ -35,7 +40,7 @@ impl Verifier for NoopVerifier {
 		Ok(())
 	}
 
-	fn verify_block_external(&self, _header: &Header, _bytes: &[u8], _engine: &Engine) -> Result<(), Error> {
+	fn verify_block_external(&self, _header: &Header, _engine: &EthEngine) -> Result<(), Error> {
 		Ok(())
 	}
 }

ethcore/src/verification/queue/kind.rs

@@ -16,7 +16,7 @@
 
 //! Definition of valid items for the verification queue.
 
-use engines::Engine;
+use engines::EthEngine;
 use error::Error;
 
 use heapsize::HeapSizeOf;
@@ -59,17 +59,17 @@ pub trait Kind: 'static + Sized + Send + Sync {
 	type Verified: Sized + Send + BlockLike + HeapSizeOf;
 
 	/// Attempt to create the `Unverified` item from the input.
-	fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error>;
+	fn create(input: Self::Input, engine: &EthEngine) -> Result<Self::Unverified, Error>;
 
 	/// Attempt to verify the `Unverified` item using the given engine.
-	fn verify(unverified: Self::Unverified, engine: &Engine, check_seal: bool) -> Result<Self::Verified, Error>;
+	fn verify(unverified: Self::Unverified, engine: &EthEngine, check_seal: bool) -> Result<Self::Verified, Error>;
 }
 
 /// The blocks verification module.
 pub mod blocks {
 	use super::{Kind, BlockLike};
 
-	use engines::Engine;
+	use engines::EthEngine;
 	use error::Error;
 	use header::Header;
 	use verification::{PreverifiedBlock, verify_block_basic, verify_block_unordered};
@@ -87,7 +87,7 @@ pub mod blocks {
 		type Unverified = Unverified;
 		type Verified = PreverifiedBlock;
 
-		fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error> {
+		fn create(input: Self::Input, engine: &EthEngine) -> Result<Self::Unverified, Error> {
 			match verify_block_basic(&input.header, &input.bytes, engine) {
 				Ok(()) => Ok(input),
 				Err(e) => {
@@ -97,7 +97,7 @@ pub mod blocks {
 			}
 		}
 
-		fn verify(un: Self::Unverified, engine: &Engine, check_seal: bool) -> Result<Self::Verified, Error> {
+		fn verify(un: Self::Unverified, engine: &EthEngine, check_seal: bool) -> Result<Self::Verified, Error> {
 			let hash = un.hash();
 			match verify_block_unordered(un.header, un.bytes, engine, check_seal) {
 				Ok(verified) => Ok(verified),
@@ -167,7 +167,7 @@ pub mod blocks {
 pub mod headers {
 	use super::{Kind, BlockLike};
 
-	use engines::Engine;
+	use engines::EthEngine;
 	use error::Error;
 	use header::Header;
 	use verification::verify_header_params;
@@ -189,13 +189,13 @@ pub mod headers {
 		type Unverified = Header;
 		type Verified = Header;
 
-		fn create(input: Self::Input, engine: &Engine) -> Result<Self::Unverified, Error> {
+		fn create(input: Self::Input, engine: &EthEngine) -> Result<Self::Unverified, Error> {
 			verify_header_params(&input, engine, true).map(|_| input)
 		}
 
-		fn verify(unverified: Self::Unverified, engine: &Engine, check_seal: bool) -> Result<Self::Verified, Error> {
+		fn verify(unverified: Self::Unverified, engine: &EthEngine, check_seal: bool) -> Result<Self::Verified, Error> {
 			match check_seal {
-				true => engine.verify_block_unordered(&unverified, None).map(|_| unverified),
+				true => engine.verify_block_unordered(&unverified,).map(|_| unverified),
 				false => Ok(unverified),
 			}
 		}

ethcore/src/verification/queue/mod.rs

@@ -28,7 +28,7 @@ use bigint::hash::H256;
 use parking_lot::{Condvar, Mutex, RwLock};
 use io::*;
 use error::*;
-use engines::Engine;
+use engines::EthEngine;
 use service::*;
 
 use self::kind::{BlockLike, Kind};
@@ -141,7 +141,7 @@ struct Sizes {
 /// A queue of items to be verified. Sits between network or other I/O and the `BlockChain`.
 /// Keeps them in the same order as inserted, minus invalid items.
 pub struct VerificationQueue<K: Kind> {
-	engine: Arc<Engine>,
+	engine: Arc<EthEngine>,
 	more_to_verify: Arc<SCondvar>,
 	verification: Arc<Verification<K>>,
 	deleting: Arc<AtomicBool>,
@@ -213,7 +213,7 @@ struct Verification<K: Kind> {
 
 impl<K: Kind> VerificationQueue<K> {
 	/// Creates a new queue instance.
-	pub fn new(config: Config, engine: Arc<Engine>, message_channel: IoChannel<ClientIoMessage>, check_seal: bool) -> Self {
+	pub fn new(config: Config, engine: Arc<EthEngine>, message_channel: IoChannel<ClientIoMessage>, check_seal: bool) -> Self {
 		let verification = Arc::new(Verification {
 			unverified: Mutex::new(VecDeque::new()),
 			verifying: Mutex::new(VecDeque::new()),
@@ -294,7 +294,7 @@ impl<K: Kind> VerificationQueue<K> {
 
 	fn verify(
 		verification: Arc<Verification<K>>,
-		engine: Arc<Engine>,
+		engine: Arc<EthEngine>,
 		wait: Arc<SCondvar>,
 		ready: Arc<QueueSignal>,
 		empty: Arc<SCondvar>,

ethcore/src/verification/verification.rs

@@ -22,20 +22,24 @@
 //! 3. Final verification against the blockchain done before enactment.
 
 use std::collections::HashSet;
-use hash::keccak;
-use triehash::ordered_trie_root;
-use heapsize::HeapSizeOf;
-use bigint::hash::H256;
-use unexpected::{Mismatch, OutOfBounds};
-use bytes::Bytes;
-use engines::Engine;
-use error::{BlockError, Error};
+
 use blockchain::*;
+use client::BlockChainClient;
+use engines::EthEngine;
+use error::{BlockError, Error};
 use header::{BlockNumber, Header};
-use rlp::UntrustedRlp;
 use transaction::SignedTransaction;
 use views::BlockView;
+
+use bigint::hash::H256;
+use bigint::prelude::U256;
+use bytes::Bytes;
+use hash::keccak;
+use heapsize::HeapSizeOf;
+use rlp::UntrustedRlp;
 use time::get_time;
+use triehash::ordered_trie_root;
+use unexpected::{Mismatch, OutOfBounds};
 
 /// Preprocessed block data gathered in `verify_block_unordered` call
 pub struct PreverifiedBlock {
@@ -56,14 +60,14 @@ impl HeapSizeOf for PreverifiedBlock {
 }
 
 /// Phase 1 quick block verification. Only does checks that are cheap. Operates on a single block
-pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> {
+pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &EthEngine) -> Result<(), Error> {
 	verify_header_params(&header, engine, true)?;
 	verify_block_integrity(bytes, &header.transactions_root(), &header.uncles_hash())?;
-	engine.verify_block_basic(&header, Some(bytes))?;
+	engine.verify_block_basic(&header)?;
 	for u in UntrustedRlp::new(bytes).at(2)?.iter().map(|rlp| rlp.as_val::<Header>()) {
 		let u = u?;
 		verify_header_params(&u, engine, false)?;
-		engine.verify_block_basic(&u, None)?;
+		engine.verify_block_basic(&u)?;
 	}
 	// Verify transactions.
 	// TODO: either use transaction views or cache the decoded transactions.
@@ -77,11 +81,11 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res
 /// Phase 2 verification. Perform costly checks such as transaction signatures and block nonce for ethash.
 /// Still operates on a individual block
 /// Returns a `PreverifiedBlock` structure populated with transactions
-pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine, check_seal: bool) -> Result<PreverifiedBlock, Error> {
+pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &EthEngine, check_seal: bool) -> Result<PreverifiedBlock, Error> {
 	if check_seal {
-		engine.verify_block_unordered(&header, Some(&bytes))?;
+		engine.verify_block_unordered(&header)?;
 		for u in UntrustedRlp::new(&bytes).at(2)?.iter().map(|rlp| rlp.as_val::<Header>()) {
-			engine.verify_block_unordered(&u?, None)?;
+			engine.verify_block_unordered(&u?)?;
 		}
 	}
 	// Verify transactions.
@@ -92,7 +96,7 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine, che
 	{
 		let v = BlockView::new(&bytes);
 		for t in v.transactions() {
-			let t = engine.verify_transaction(t, &header)?;
+			let t = engine.verify_transaction_unordered(t, &header)?;
 			if let Some(max_nonce) = nonce_cap {
 				if t.nonce >= max_nonce {
 					return Err(BlockError::TooManyTransactions(t.sender()).into());
@@ -108,13 +112,30 @@ pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine, che
 	})
 }
 
-/// Phase 3 verification. Check block information against parent and uncles.
-pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error> {
-	// TODO: verify timestamp
-	let parent = bc.block_header(&header.parent_hash()).ok_or_else(|| Error::from(BlockError::UnknownParent(header.parent_hash().clone())))?;
-	verify_parent(&header, &parent)?;
-	engine.verify_block_family(&header, &parent, Some(bytes))?;
+/// Parameters for full verification of block family: block bytes, transactions, blockchain, and state access.
+pub type FullFamilyParams<'a> = (&'a [u8], &'a [SignedTransaction], &'a BlockProvider, &'a BlockChainClient);
 
+/// Phase 3 verification. Check block information against parent and uncles.
+pub fn verify_block_family(header: &Header, parent: &Header, engine: &EthEngine, do_full: Option<FullFamilyParams>) -> Result<(), Error> {
+	// TODO: verify timestamp
+	verify_parent(&header, &parent, engine.params().gas_limit_bound_divisor)?;
+	engine.verify_block_family(&header, &parent)?;
+
+	let (bytes, txs, bc, client) = match do_full {
+		Some(x) => x,
+		None => return Ok(()),
+	};
+
+	verify_uncles(header, bytes, bc, engine)?;
+
+	for transaction in txs {
+		engine.machine().verify_transaction(transaction, header, client)?;
+	}
+
+	Ok(())
+}
+
+fn verify_uncles(header: &Header, bytes: &[u8], bc: &BlockProvider, engine: &EthEngine) -> Result<(), Error> {
 	let num_uncles = UntrustedRlp::new(bytes).at(2)?.item_count()?;
 	if num_uncles != 0 {
 		if num_uncles > engine.maximum_uncle_count() {
@@ -189,11 +210,12 @@ pub fn verify_block_family(header: &Header, bytes: &[u8], engine: &Engine, bc: &
 				return Err(From::from(BlockError::UncleParentNotInChain(uncle_parent.hash())));
 			}
 
-			verify_parent(&uncle, &uncle_parent)?;
-			engine.verify_block_family(&uncle, &uncle_parent, Some(bytes))?;
+			verify_parent(&uncle, &uncle_parent, engine.params().gas_limit_bound_divisor)?;
+			engine.verify_block_family(&uncle, &uncle_parent)?;
 			verified.insert(uncle.hash());
 		}
 	}
+
 	Ok(())
 }
 
@@ -215,7 +237,13 @@ pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>
 }
 
 /// Check basic header parameters.
-pub fn verify_header_params(header: &Header, engine: &Engine, is_full: bool) -> Result<(), Error> {
+pub fn verify_header_params(header: &Header, engine: &EthEngine, is_full: bool) -> Result<(), Error> {
+	if header.seal().len() != engine.seal_fields() {
+		return Err(From::from(BlockError::InvalidSealArity(
+			Mismatch { expected: engine.seal_fields(), found: header.seal().len() }
+		)));
+	}
+
 	if header.number() >= From::from(BlockNumber::max_value()) {
 		return Err(From::from(BlockError::RidiculousNumber(OutOfBounds { max: Some(From::from(BlockNumber::max_value())), min: None, found: header.number() })))
 	}
@@ -230,6 +258,15 @@ pub fn verify_header_params(header: &Header, engine: &Engine, is_full: bool) ->
 	if header.number() != 0 && header.extra_data().len() > maximum_extra_data_size {
 		return Err(From::from(BlockError::ExtraDataOutOfBounds(OutOfBounds { min: None, max: Some(maximum_extra_data_size), found: header.extra_data().len() })));
 	}
+
+	if let Some(ref ext) = engine.machine().ethash_extensions() {
+		if header.number() >= ext.dao_hardfork_transition &&
+			header.number() <= ext.dao_hardfork_transition + 9 &&
+			header.extra_data()[..] != b"dao-hard-fork"[..] {
+			return Err(From::from(BlockError::ExtraDataOutOfBounds(OutOfBounds { min: None, max: None, found: 0 })));
+		}
+	}
+
 	if is_full {
 		let max_time = get_time().sec as u64 + 30;
 		if header.timestamp() > max_time {
@@ -240,7 +277,7 @@ pub fn verify_header_params(header: &Header, engine: &Engine, is_full: bool) ->
 }
 
 /// Check header parameters agains parent header.
-fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
+fn verify_parent(header: &Header, parent: &Header, gas_limit_divisor: U256) -> Result<(), Error> {
 	if !header.parent_hash().is_zero() && &parent.hash() != header.parent_hash() {
 		return Err(From::from(BlockError::InvalidParentHash(Mismatch { expected: parent.hash(), found: header.parent_hash().clone() })))
 	}
@@ -250,6 +287,18 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
 	if header.number() != parent.number() + 1 {
 		return Err(From::from(BlockError::InvalidNumber(Mismatch { expected: parent.number() + 1, found: header.number() })));
 	}
+
+	if header.number() == 0 {
+		return Err(BlockError::RidiculousNumber(OutOfBounds { min: Some(1), max: None, found: header.number() }).into());
+	}
+
+	let parent_gas_limit = *parent.gas_limit();
+	let min_gas = parent_gas_limit - parent_gas_limit / gas_limit_divisor;
+	let max_gas = parent_gas_limit + parent_gas_limit / gas_limit_divisor;
+	if header.gas_limit() <= &min_gas || header.gas_limit() >= &max_gas {
+		return Err(From::from(BlockError::InvalidGasLimit(OutOfBounds { min: Some(min_gas), max: Some(max_gas), found: header.gas_limit().clone() })));
+	}
+
 	Ok(())
 }
 
@@ -285,7 +334,7 @@ mod tests {
 	use error::BlockError::*;
 	use views::*;
 	use blockchain::*;
-	use engines::Engine;
+	use engines::EthEngine;
 	use spec::*;
 	use transaction::*;
 	use tests::helpers::*;
@@ -406,17 +455,38 @@ mod tests {
 		}
 	}
 
-	fn basic_test(bytes: &[u8], engine: &Engine) -> Result<(), Error> {
+	fn basic_test(bytes: &[u8], engine: &EthEngine) -> Result<(), Error> {
 		let header = BlockView::new(bytes).header();
 		verify_block_basic(&header, bytes, engine)
 	}
 
-	fn family_test<BC>(bytes: &[u8], engine: &Engine, bc: &BC) -> Result<(), Error> where BC: BlockProvider {
-		let header = BlockView::new(bytes).header();
-		verify_block_family(&header, bytes, engine, bc)
+	fn family_test<BC>(bytes: &[u8], engine: &EthEngine, bc: &BC) -> Result<(), Error> where BC: BlockProvider {
+		let view = BlockView::new(bytes);
+		let header = view.header();
+		let transactions: Vec<_> = view.transactions()
+			.into_iter()
+			.map(SignedTransaction::new)
+			.collect::<Result<_,_>>()?;
+
+		// TODO: client is really meant to be used for state query here by machine
+		// additions that need access to state (tx filter in specific)
+		// no existing tests need access to test, so having this not function
+		// is fine.
+		let client = ::client::TestBlockChainClient::default();
+
+		let parent = bc.block_header(header.parent_hash())
+			.ok_or(BlockError::UnknownParent(header.parent_hash().clone()))?;
+
+		let full_params: FullFamilyParams = (
+			bytes,
+			&transactions[..],
+			bc as &BlockProvider,
+			&client as &::client::BlockChainClient
+		);
+		verify_block_family(&header, &parent, engine, Some(full_params))
 	}
 
-	fn unordered_test(bytes: &[u8], engine: &Engine) -> Result<(), Error> {
+	fn unordered_test(bytes: &[u8], engine: &EthEngine) -> Result<(), Error> {
 		let header = BlockView::new(bytes).header();
 		verify_block_unordered(header, bytes.to_vec(), engine, false)?;
 		Ok(())
@@ -590,6 +660,15 @@ mod tests {
 		check_fail(family_test(&create_test_block_with_data(&header, &good_transactions, &bad_uncles), engine, &bc),
 			DuplicateUncle(good_uncle1.hash()));
 
+		header = good.clone();
+		header.set_gas_limit(0.into());
+		header.set_difficulty("0000000000000000000000000000000000000000000000000000000000020000".parse::<U256>().unwrap());
+		match family_test(&create_test_block(&header), engine, &bc) {
+			Err(Error::Block(InvalidGasLimit(_))) => {},
+			Err(_) => { panic!("should be invalid difficulty fail"); },
+			_ => { panic!("Should be error, got Ok"); },
+		}
+
 		// TODO: some additional uncle checks
 	}
 
@@ -597,6 +676,7 @@ mod tests {
 	fn dust_protection() {
 		use ethkey::{Generator, Random};
 		use transaction::{Transaction, Action};
+		use machine::EthereumMachine;
 		use engines::NullEngine;
 
 		let mut params = CommonParams::default();
@@ -618,7 +698,8 @@ mod tests {
 
 		let good_transactions = [bad_transactions[0].clone(), bad_transactions[1].clone()];
 
-		let engine = NullEngine::new(params, BTreeMap::new());
+		let machine = EthereumMachine::regular(params, BTreeMap::new());
+		let engine = NullEngine::new(Default::default(), machine);
 		check_fail(unordered_test(&create_test_block_with_data(&header, &bad_transactions, &[]), &engine), TooManyTransactions(keypair.address()));
 		unordered_test(&create_test_block_with_data(&header, &good_transactions, &[]), &engine).unwrap();
 	}

ethcore/src/verification/verifier.rs

@@ -16,17 +16,24 @@
 
 //! A generic verifier trait.
 
-use blockchain::BlockProvider;
-use engines::Engine;
+use engines::EthEngine;
 use error::Error;
 use header::Header;
+use super::verification;
 
 /// Should be used to verify blocks.
 pub trait Verifier: Send + Sync {
 	/// Verify a block relative to its parent and uncles.
-	fn verify_block_family(&self, header: &Header, bytes: &[u8], engine: &Engine, bc: &BlockProvider) -> Result<(), Error>;
+    fn verify_block_family(
+		&self,
+		header: &Header,
+		parent: &Header,
+		engine: &EthEngine,
+		do_full: Option<verification::FullFamilyParams>
+	) -> Result<(), Error>;
+
 	/// Do a final verification check for an enacted header vs its expected counterpart.
 	fn verify_block_final(&self, expected: &Header, got: &Header) -> Result<(), Error>;
 	/// Verify a block, inspecing external state.
-	fn verify_block_external(&self, header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error>;
+	fn verify_block_external(&self, header: &Header, engine: &EthEngine) -> Result<(), Error>;
 }