Benchmarks for block verification (#11035)

* WIP

* wip

* Benchmarks for block verification

Uses real blocks from mainnet to benchmark the `verify_*` family of methods in the `verification` module.

Also exposes the `TestBlockChain` in a test helper.

* Cleanup, fix CI

* Bash syntax error

* One more try

* Fix review grumbles
	Revert unwanted changes
	Tweak CI benchmark checks

.gitlab-ci.yml

@@ -100,10 +100,9 @@ cargo-check-benches:
   stage:                           test
   <<:                              *docker-cache-status
   script:
-    - time (
+    - time (cargo check --all --benches --exclude ethash --exclude verification --target $CARGO_TARGET --locked --verbose --color=always)
-        cargo check --all --benches --exclude ethash --target $CARGO_TARGET --locked --verbose --color=always;
+    - time (cd ethash; cargo check --benches --features bench --target $CARGO_TARGET --locked --verbose --color=always)
-        (cd ethash; time cargo check --benches --features bench --target $CARGO_TARGET --locked --verbose --color=always)
+    - time (cd ethcore/verification; cargo check --benches --features bench --target $CARGO_TARGET --locked --verbose --color=always)
-      )
     - sccache -s
 
 cargo-audit:

Cargo.lock

@@ -4996,7 +4996,9 @@ version = "0.1.0"
 dependencies = [
  "client-traits 0.1.0",
  "common-types 0.1.0",
+ "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "engine 0.1.0",
+ "ethash-engine 0.1.0",
  "ethcore 1.12.0",
  "ethcore-blockchain 0.1.0",
  "ethcore-call-contract 0.1.0",
@@ -5014,6 +5016,7 @@ dependencies = [
  "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "spec 0.1.0",
+ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "time-utils 0.1.0",
  "triehash-ethereum 0.2.0",
  "unexpected 0.1.0",

ethcore/types/src/verification.rs

@@ -57,7 +57,7 @@ impl VerificationQueueInfo {
 }
 
 /// An unverified block.
-#[derive(PartialEq, Debug, MallocSizeOf)]
+#[derive(Clone, PartialEq, Debug, MallocSizeOf)]
 pub struct Unverified {
 	/// Unverified block header.
 	pub header: Header,

ethcore/verification/Cargo.toml

@@ -6,6 +6,10 @@ authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 license = "GPL-3.0"
 
+[[bench]]
+name = "verification"
+harness = false
+
 [dependencies]
 blockchain = { package = "ethcore-blockchain", path = "../blockchain" }
 call-contract = { package = "ethcore-call-contract", path = "../call-contract" }
@@ -27,8 +31,17 @@ triehash = { package = "triehash-ethereum", version = "0.2",  path = "../../util
 unexpected = { path = "../../util/unexpected" }
 
 [dev-dependencies]
+criterion = "0.3"
 ethcore = { path = "../", features = ["test-helpers"] }
 ethkey = { path = "../../accounts/ethkey" }
 machine = { path = "../machine" }
 null-engine = { path = "../engines/null-engine" }
 spec = { path = "../spec" }
+
+# Benches
+ethash = { package = "ethash-engine", path = "../engines/ethash" }
+tempdir = "0.3.7"
+
+[features]
+# Used to selectively expose code for benchmarks.
+bench = []

ethcore/verification/benches/verification.rs

@@ -0,0 +1,161 @@
+// Copyright 2015-2019 Parity Technologies (UK) Ltd.
+// This file is part of Parity Ethereum.
+
+// Parity Ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
+
+//! benchmarking for verification
+
+use std::collections::BTreeMap;
+
+use common_types::verification::Unverified;
+use criterion::{Criterion, criterion_group, criterion_main};
+use ethash::{EthashParams, Ethash};
+use ethereum_types::U256;
+use ethcore::client::TestBlockChainClient;
+use spec::new_constantinople_test_machine;
+use tempdir::TempDir;
+
+use ::verification::{
+	FullFamilyParams,
+	verification,
+	test_helpers::TestBlockChain,
+};
+
+// These are current production values. Needed when using real blocks.
+fn ethash_params() -> EthashParams {
+	EthashParams {
+		minimum_difficulty: U256::from(131072),
+		difficulty_bound_divisor: U256::from(2048),
+		difficulty_increment_divisor: 10,
+		metropolis_difficulty_increment_divisor: 9,
+		duration_limit: 13,
+		homestead_transition: 1150000,
+		difficulty_hardfork_transition: u64::max_value(),
+		difficulty_hardfork_bound_divisor: U256::from(2048),
+		bomb_defuse_transition: u64::max_value(),
+		eip100b_transition: 4370000,
+		ecip1010_pause_transition: u64::max_value(),
+		ecip1010_continue_transition: u64::max_value(),
+		ecip1017_era_rounds: u64::max_value(),
+		block_reward: {
+			let mut m = BTreeMap::<u64, U256>::new();
+			m.insert(0, 5000000000000000000u64.into());
+			m.insert(4370000, 3000000000000000000u64.into());
+			m.insert(7280000, 2000000000000000000u64.into());
+			m
+		},
+		expip2_transition: u64::max_value(),
+		expip2_duration_limit: 30,
+		block_reward_contract_transition: 0,
+		block_reward_contract: None,
+		difficulty_bomb_delays: {
+			let mut m = BTreeMap::new();
+			m.insert(4370000, 3000000);
+			m.insert(7280000, 2000000);
+			m
+		},
+		progpow_transition: u64::max_value()
+	}
+}
+
+fn build_ethash() -> Ethash {
+	let machine = new_constantinople_test_machine();
+	let ethash_params = ethash_params();
+	let cache_dir = TempDir::new("").unwrap();
+	Ethash::new(
+		cache_dir.path(),
+		ethash_params,
+		machine,
+		None
+	)
+}
+
+fn block_verification(c: &mut Criterion) {
+	const PROOF: &str = "bytes from disk are ok";
+
+	let ethash = build_ethash();
+
+	// A fairly large block (32kb) with one uncle
+	let rlp_8481476 = include_bytes!("./8481476-one-uncle.rlp").to_vec();
+	// Parent of #8481476
+	let rlp_8481475 = include_bytes!("./8481475.rlp").to_vec();
+	// Parent of the uncle in #8481476
+	let rlp_8481474 = include_bytes!("./8481474-parent-to-uncle.rlp").to_vec();
+
+	// Phase 1 verification
+	c.bench_function("verify_block_basic", |b| {
+		let block = Unverified::from_rlp(rlp_8481476.clone()).expect(PROOF);
+		b.iter(|| {
+			assert!(verification::verify_block_basic(
+				&block,
+				&ethash,
+				true
+			).is_ok());
+		})
+	});
+
+	// Phase 2 verification
+	c.bench_function("verify_block_unordered", |b| {
+		let block = Unverified::from_rlp(rlp_8481476.clone()).expect(PROOF);
+		b.iter( || {
+			assert!(verification::verify_block_unordered(
+				block.clone(),
+				&ethash,
+				true
+			).is_ok());
+		})
+	});
+
+	// Phase 3 verification
+	let block = Unverified::from_rlp(rlp_8481476.clone()).expect(PROOF);
+	let preverified = verification::verify_block_unordered(block, &ethash, true).expect(PROOF);
+	let parent = Unverified::from_rlp(rlp_8481475.clone()).expect(PROOF);
+
+	// "partial" means we skip uncle and tx verification
+	c.bench_function("verify_block_family (partial)", |b| {
+		b.iter(|| {
+			if let Err(e) = verification::verify_block_family::<TestBlockChainClient>(
+				&preverified.header,
+				&parent.header,
+				&ethash,
+				None
+			) {
+				panic!("verify_block_family (partial) ERROR: {:?}", e);
+			}
+		});
+	});
+
+	let mut block_provider = TestBlockChain::new();
+	block_provider.insert(rlp_8481476.clone()); // block to verify
+	block_provider.insert(rlp_8481475.clone()); // parent
+	block_provider.insert(rlp_8481474.clone()); // uncle's parent
+
+	let client = TestBlockChainClient::default();
+	c.bench_function("verify_block_family (full)", |b| {
+		b.iter(|| {
+			let full = FullFamilyParams { block: &preverified, block_provider: &block_provider, client: &client };
+			if let Err(e) = verification::verify_block_family::<TestBlockChainClient>(
+				&preverified.header,
+				&parent.header,
+				&ethash,
+				Some(full),
+			) {
+				panic!("verify_block_family (full) ERROR: {:?}", e)
+			}
+		});
+	});
+}
+
+criterion_group!(benches, block_verification);
+criterion_main!(benches);

ethcore/verification/src/lib.rs

@@ -21,11 +21,16 @@ use client_traits::BlockInfo;
 // The MallocSizeOf derive looks for this in the root
 use parity_util_mem as malloc_size_of;
 
+#[cfg(feature = "bench" )]
+pub mod verification;
+#[cfg(not(feature = "bench" ))]
 mod verification;
 mod verifier;
 pub mod queue;
 mod canon_verifier;
 mod noop_verifier;
+#[cfg(any(test, feature = "bench" ))]
+pub mod test_helpers;
 
 pub use self::verification::FullFamilyParams;
 pub use self::verifier::Verifier;

ethcore/verification/src/test_helpers.rs

@@ -0,0 +1,114 @@
+// Copyright 2015-2019 Parity Technologies (UK) Ltd.
+// This file is part of Parity Ethereum.
+
+// Parity Ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity Ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Verification test helpers.
+
+use std::collections::HashMap;
+
+use blockchain::{BlockProvider, BlockChain, BlockDetails, TransactionAddress, BlockReceipts};
+use common_types::{
+	BlockNumber,
+	encoded,
+	verification::Unverified,
+	log_entry::{LogEntry, LocalizedLogEntry},
+};
+use ethereum_types::{BloomRef, H256};
+use parity_bytes::Bytes;
+
+#[derive(Default)]
+pub struct TestBlockChain {
+	blocks: HashMap<H256, Bytes>,
+	numbers: HashMap<BlockNumber, H256>,
+}
+
+impl TestBlockChain {
+	pub fn new() -> Self { TestBlockChain::default() }
+
+	pub fn insert(&mut self, bytes: Bytes) {
+		let header = Unverified::from_rlp(bytes.clone()).unwrap().header;
+		let hash = header.hash();
+		self.blocks.insert(hash, bytes);
+		self.numbers.insert(header.number(), hash);
+	}
+}
+
+impl BlockProvider for TestBlockChain {
+	fn is_known(&self, hash: &H256) -> bool {
+		self.blocks.contains_key(hash)
+	}
+
+	fn first_block(&self) -> Option<H256> {
+		unimplemented!()
+	}
+
+	fn best_ancient_block(&self) -> Option<H256> {
+		None
+	}
+
+	/// Get raw block data
+	fn block(&self, hash: &H256) -> Option<encoded::Block> {
+		self.blocks.get(hash).cloned().map(encoded::Block::new)
+	}
+
+	/// Get the familial details concerning a block.
+	fn block_details(&self, hash: &H256) -> Option<BlockDetails> {
+		self.blocks.get(hash).map(|bytes| {
+			let header = Unverified::from_rlp(bytes.to_vec()).unwrap().header;
+			BlockDetails {
+				number: header.number(),
+				total_difficulty: *header.difficulty(),
+				parent: *header.parent_hash(),
+				children: Vec::new(),
+				is_finalized: false,
+			}
+		})
+	}
+
+	/// Get the hash of given block's number.
+	fn block_hash(&self, index: BlockNumber) -> Option<H256> {
+		self.numbers.get(&index).cloned()
+	}
+
+	fn transaction_address(&self, _hash: &H256) -> Option<TransactionAddress> {
+		unimplemented!()
+	}
+
+	fn block_receipts(&self, _hash: &H256) -> Option<BlockReceipts> {
+		unimplemented!()
+	}
+
+	fn block_header_data(&self, hash: &H256) -> Option<encoded::Header> {
+		self.block(hash)
+			.map(|b| b.header_view().rlp().as_raw().to_vec())
+			.map(encoded::Header::new)
+	}
+
+	fn block_body(&self, hash: &H256) -> Option<encoded::Body> {
+		self.block(hash)
+			.map(|b| BlockChain::block_to_body(&b.into_inner()))
+			.map(encoded::Body::new)
+	}
+
+	fn blocks_with_bloom<'a, B, I, II>(&self, _blooms: II, _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber>
+		where BloomRef<'a>: From<B>, II: IntoIterator<Item = B, IntoIter = I> + Copy, I: Iterator<Item = B>, Self: Sized {
+		unimplemented!()
+	}
+
+	fn logs<F>(&self, _blocks: Vec<H256>, _matches: F, _limit: Option<usize>) -> Vec<LocalizedLogEntry>
+		where F: Fn(&LogEntry) -> bool, Self: Sized {
+		unimplemented!()
+	}
+}

ethcore/verification/src/verification.rs

@@ -119,7 +119,12 @@ pub struct FullFamilyParams<'a, C: BlockInfo + CallContract + 'a> {
 }
 
 /// Phase 3 verification. Check block information against parent and uncles.
-pub fn verify_block_family<C: BlockInfo + CallContract>(header: &Header, parent: &Header, engine: &dyn Engine, do_full: Option<FullFamilyParams<C>>) -> Result<(), Error> {
+pub fn verify_block_family<C: BlockInfo + CallContract>(
+	header: &Header,
+	parent: &Header,
+	engine: &dyn Engine,
+	do_full: Option<FullFamilyParams<C>>
+) -> Result<(), Error> {
 	// TODO: verify timestamp
 	verify_parent(&header, &parent, engine)?;
 	engine.verify_block_family(&header, &parent)?;
@@ -128,7 +133,6 @@ pub fn verify_block_family<C: BlockInfo + CallContract>(header: &Header, parent:
 		Some(x) => x,
 		None => return Ok(()),
 	};
-
 	verify_uncles(params.block, params.block_provider, engine)?;
 
 	for tx in &params.block.transactions {
@@ -248,7 +252,7 @@ pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error>
 }
 
 /// Check basic header parameters.
-pub fn verify_header_params(header: &Header, engine: &dyn Engine, is_full: bool, check_seal: bool) -> Result<(), Error> {
+pub(crate) fn verify_header_params(header: &Header, engine: &dyn Engine, is_full: bool, check_seal: bool) -> Result<(), Error> {
 	if check_seal {
 		let expected_seal_fields = engine.seal_fields(header);
 		if header.seal().len() != expected_seal_fields {
@@ -306,7 +310,7 @@ pub fn verify_header_params(header: &Header, engine: &dyn Engine, is_full: bool,
 	Ok(())
 }
 
-/// Check header parameters agains parent header.
+/// Check header parameters against parent header.
 fn verify_parent(header: &Header, parent: &Header, engine: &dyn Engine) -> Result<(), Error> {
 	assert!(header.parent_hash().is_zero() || &parent.hash() == header.parent_hash(),
 			"Parent hash should already have been verified; qed");
@@ -364,11 +368,10 @@ fn verify_block_integrity(block: &Unverified) -> Result<(), Error> {
 mod tests {
 	use super::*;
 
-	use std::collections::{BTreeMap, HashMap};
+	use std::collections::BTreeMap;
 	use std::time::{SystemTime, UNIX_EPOCH};
 
-	use ethereum_types::{H256, BloomRef, U256, Address};
+	use ethereum_types::{H256, U256, Address};
-	use blockchain::{BlockDetails, TransactionAddress, BlockReceipts};
 	use parity_bytes::Bytes;
 	use keccak_hash::keccak;
 	use engine::Engine;
@@ -379,17 +382,17 @@ mod tests {
 		test_helpers::{create_test_block_with_data, create_test_block}
 	};
 	use common_types::{
-		encoded,
 		engines::params::CommonParams,
 		errors::BlockError::*,
 		transaction::{SignedTransaction, Transaction, UnverifiedTransaction, Action},
-		log_entry::{LogEntry, LocalizedLogEntry},
 	};
 	use rlp;
 	use triehash::ordered_trie_root;
 	use machine::Machine;
 	use null_engine::NullEngine;
 
+	use crate::test_helpers::TestBlockChain;
+
 	fn check_ok(result: Result<(), Error>) {
 		result.unwrap_or_else(|e| panic!("Block verification failed: {:?}", e));
 	}
@@ -412,101 +415,6 @@ mod tests {
 		}
 	}
 
-	struct TestBlockChain {
-		blocks: HashMap<H256, Bytes>,
-		numbers: HashMap<BlockNumber, H256>,
-	}
-
-	impl Default for TestBlockChain {
-		fn default() -> Self {
-			TestBlockChain::new()
-		}
-	}
-
-	impl TestBlockChain {
-		pub fn new() -> Self {
-			TestBlockChain {
-				blocks: HashMap::new(),
-				numbers: HashMap::new(),
-			}
-		}
-
-		pub fn insert(&mut self, bytes: Bytes) {
-			let header = Unverified::from_rlp(bytes.clone()).unwrap().header;
-			let hash = header.hash();
-			self.blocks.insert(hash, bytes);
-			self.numbers.insert(header.number(), hash);
-		}
-	}
-
-	impl BlockProvider for TestBlockChain {
-		fn is_known(&self, hash: &H256) -> bool {
-			self.blocks.contains_key(hash)
-		}
-
-		fn first_block(&self) -> Option<H256> {
-			unimplemented!()
-		}
-
-		/// Get raw block data
-		fn block(&self, hash: &H256) -> Option<encoded::Block> {
-			self.blocks.get(hash).cloned().map(encoded::Block::new)
-		}
-
-		fn block_header_data(&self, hash: &H256) -> Option<encoded::Header> {
-			self.block(hash)
-				.map(|b| b.header_view().rlp().as_raw().to_vec())
-				.map(encoded::Header::new)
-		}
-
-		fn block_body(&self, hash: &H256) -> Option<encoded::Body> {
-			self.block(hash)
-				.map(|b| BlockChain::block_to_body(&b.into_inner()))
-				.map(encoded::Body::new)
-		}
-
-		fn best_ancient_block(&self) -> Option<H256> {
-			None
-		}
-
-		/// Get the familial details concerning a block.
-		fn block_details(&self, hash: &H256) -> Option<BlockDetails> {
-			self.blocks.get(hash).map(|bytes| {
-				let header = Unverified::from_rlp(bytes.to_vec()).unwrap().header;
-				BlockDetails {
-					number: header.number(),
-					total_difficulty: *header.difficulty(),
-					parent: *header.parent_hash(),
-					children: Vec::new(),
-					is_finalized: false,
-				}
-			})
-		}
-
-		fn transaction_address(&self, _hash: &H256) -> Option<TransactionAddress> {
-			unimplemented!()
-		}
-
-		/// Get the hash of given block's number.
-		fn block_hash(&self, index: BlockNumber) -> Option<H256> {
-			self.numbers.get(&index).cloned()
-		}
-
-		fn block_receipts(&self, _hash: &H256) -> Option<BlockReceipts> {
-			unimplemented!()
-		}
-
-		fn blocks_with_bloom<'a, B, I, II>(&self, _blooms: II, _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber>
-		where BloomRef<'a>: From<B>, II: IntoIterator<Item = B, IntoIter = I> + Copy, I: Iterator<Item = B>, Self: Sized {
-			unimplemented!()
-		}
-
-		fn logs<F>(&self, _blocks: Vec<H256>, _matches: F, _limit: Option<usize>) -> Vec<LocalizedLogEntry>
-			where F: Fn(&LogEntry) -> bool, Self: Sized {
-			unimplemented!()
-		}
-	}
-
 	fn basic_test(bytes: &[u8], engine: &dyn Engine) -> Result<(), Error> {
 		let unverified = Unverified::from_rlp(bytes.to_vec())?;
 		verify_block_basic(&unverified, engine, true)