EVM benchmark utilities (#8944)

* Make it possible to expose jsontests using feature flag

* Add start_stop_hook for jsontests

* Fix evmbin compile

* Implement vm jsontests times to tsv result

* Use /usr/bin/env to avoid errors on non-Debian systems

* Move evmbin/bench.sh to scripts and add vm_jsontests script for convenience

* Add tempdir as required deps for test-helpers

* Address grumbles on comments

* Detect file/folder automatically and add command docs

* Fix bench script

* times -> timings

* typo: wrong if condition

ethcore/Cargo.toml

@@ -67,7 +67,7 @@ keccak-hash = { path = "../util/hash" }
 triehash = { path = "../util/triehash" }
 unexpected = { path = "../util/unexpected" }
 journaldb = { path = "../util/journaldb" }
-tempdir = "0.3"
+tempdir = { version = "0.3", optional = true }
 kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 
 [dev-dependencies]
@@ -84,10 +84,10 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["ethcore-transaction/json-tests"]
+json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"]
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile benches
 benches = []
 # Compile test helpers
-test-helpers = []
+test-helpers = ["tempdir"]

ethcore/src/client/evm_test_client.rs

@@ -64,15 +64,6 @@ impl fmt::Display for EvmTestError {
 use ethereum;
 use ethjson::state::test::ForkSpec;
 
-lazy_static! {
-	pub static ref FRONTIER: spec::Spec = ethereum::new_frontier_test();
-	pub static ref HOMESTEAD: spec::Spec = ethereum::new_homestead_test();
-	pub static ref EIP150: spec::Spec = ethereum::new_eip150_test();
-	pub static ref EIP161: spec::Spec = ethereum::new_eip161_test();
-	pub static ref BYZANTIUM: spec::Spec = ethereum::new_byzantium_test();
-	pub static ref BYZANTIUM_TRANSITION: spec::Spec = ethereum::new_transition_test();
-}
-
 /// Simplified, single-block EVM test client.
 pub struct EvmTestClient<'a> {
 	state: state::State<state_db::StateDB>,
@@ -90,14 +81,14 @@ impl<'a> fmt::Debug for EvmTestClient<'a> {
 
 impl<'a> EvmTestClient<'a> {
 	/// Converts a json spec definition into spec.
-	pub fn spec_from_json(spec: &ForkSpec) -> Option<&'static spec::Spec> {
+	pub fn spec_from_json(spec: &ForkSpec) -> Option<spec::Spec> {
 		match *spec {
-			ForkSpec::Frontier => Some(&*FRONTIER),
-			ForkSpec::Homestead => Some(&*HOMESTEAD),
-			ForkSpec::EIP150 => Some(&*EIP150),
-			ForkSpec::EIP158 => Some(&*EIP161),
-			ForkSpec::Byzantium => Some(&*BYZANTIUM),
-			ForkSpec::EIP158ToByzantiumAt5 => Some(&BYZANTIUM_TRANSITION),
+			ForkSpec::Frontier => Some(ethereum::new_frontier_test()),
+			ForkSpec::Homestead => Some(ethereum::new_homestead_test()),
+			ForkSpec::EIP150 => Some(ethereum::new_eip150_test()),
+			ForkSpec::EIP158 => Some(ethereum::new_eip161_test()),
+			ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()),
+			ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()),
 			ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None,
 			_ => None,
 		}

ethcore/src/json_tests/chain.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use std::sync::Arc;
 use client::{EvmTestClient, Client, ClientConfig, ChainInfo, ImportBlock};
 use block::Block;
@@ -23,12 +24,26 @@ use miner::Miner;
 use io::IoChannel;
 use test_helpers;
 
-pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
+use super::HookType;
+
+/// Run chain jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
+}
+
+/// Run chain jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, json_chain_test, h)
+}
+
+pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::blockchain::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		let mut fail = false;
 		{
 			let mut fail_unless = |cond: bool| if !cond && !fail {
@@ -42,7 +57,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 
 			let spec = {
 				let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) {
-					Some(spec) => (*spec).clone(),
+					Some(spec) => spec,
 					None => {
 						println!("   - {} | {:?} Ignoring tests because of missing spec", name, blockchain.network);
 						continue;
@@ -82,17 +97,21 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 		if !fail {
 			flushln!("ok");
 		}
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	println!("!!! {:?} tests from failed.", failed.len());
 	failed
 }
 
+#[cfg(test)]
 mod block_tests {
 	use super::json_chain_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_chain_test(json_data)
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_chain_test(json_data, h)
 	}
 
 	declare_test!{BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"}

ethcore/src/json_tests/difficulty.rs

@@ -19,12 +19,16 @@ use header::Header;
 use ethereum_types::U256;
 use spec::Spec;
 
-pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec<String> {
+use super::HookType;
+
+pub fn json_difficulty_test<H: FnMut(&str, HookType)>(json_data: &[u8], spec: Spec, start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::test::DifficultyTest::load(json_data).unwrap();
 	let engine = &spec.engine;
 
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		flush!("   - {}...", name);
 		println!("   - {}...", name);
 
@@ -42,15 +46,18 @@ pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec<String> {
 		let expected_difficulty: U256 = test.current_difficulty.into();
 		assert_eq!(header.difficulty(), &expected_difficulty);
 		flushln!("ok");
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 	vec![]
 }
 
 mod difficulty_test_byzantium {
 	use super::json_difficulty_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_difficulty_test(json_data, ::ethereum::new_byzantium_test())
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_difficulty_test(json_data, ::ethereum::new_byzantium_test(), h)
 	}
 
 	declare_test!{DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"}
@@ -59,10 +66,11 @@ mod difficulty_test_byzantium {
 mod difficulty_test_foundation {
 	use super::json_difficulty_test;
 	use tempdir::TempDir;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
 		let tempdir = TempDir::new("").unwrap();
-		json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()))
+		json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()), h)
 	}
 
 	declare_test!{DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"}

ethcore/src/json_tests/executive.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use std::sync::Arc;
 use super::test_common::*;
 use state::{Backend as StateBackend, State, Substate};
@@ -35,6 +36,18 @@ use rlp::RlpStream;
 use hash::keccak;
 use machine::EthereumMachine as Machine;
 
+use super::HookType;
+
+/// Run executive jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+}
+
+/// Run executive jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, do_json_test, h)
+}
+
 #[derive(Debug, PartialEq, Clone)]
 struct CallCreate {
 	data: Bytes,
@@ -193,20 +206,22 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B>
 	}
 }
 
-fn do_json_test(json_data: &[u8]) -> Vec<String> {
+fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
 	let vms = VMType::all();
 	vms
 		.iter()
-		.flat_map(|vm| do_json_test_for(vm, json_data))
+		.flat_map(|vm| do_json_test_for(vm, json_data, h))
 		.collect()
 }
 
-fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
+fn do_json_test_for<H: FnMut(&str, HookType)>(vm_type: &VMType, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	let tests = ethjson::vm::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, vm) in tests.into_iter() {
-		println!("name: {:?}", name);
+		start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart);
+
+		info!(target: "jsontests", "name: {:?}", name);
 		let mut fail = false;
 
 		let mut fail_unless = |cond: bool, s: &str | if !cond && !fail {
@@ -305,10 +320,12 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec<String> {
 				fail_unless(Some(callcreates) == calls, "callcreates does not match");
 			}
 		};
+
+		start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop);
 	}
 
 	for f in &failed {
-		println!("FAILED: {:?}", f);
+		error!("FAILED: {:?}", f);
 	}
 
 	failed

ethcore/src/json_tests/mod.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+//! Helpers and tests for operating on jsontests.
+
 #[macro_use]
 mod test_common;
 
@@ -22,4 +24,21 @@ mod executive;
 mod state;
 mod chain;
 mod trie;
+
+#[cfg(test)]
 mod difficulty;
+
+pub use self::test_common::HookType;
+
+pub use self::transaction::run_test_path as run_transaction_test_path;
+pub use self::transaction::run_test_file as run_transaction_test_file;
+pub use self::executive::run_test_path as run_executive_test_path;
+pub use self::executive::run_test_file as run_executive_test_file;
+pub use self::state::run_test_path as run_state_test_path;
+pub use self::state::run_test_file as run_state_test_file;
+pub use self::chain::run_test_path as run_chain_test_path;
+pub use self::chain::run_test_file as run_chain_test_file;
+pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
+pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
+pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
+pub use self::trie::run_secure_test_file as run_secure_trie_test_file;

ethcore/src/json_tests/state.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use super::test_common::*;
 use pod_state::PodState;
 use trace;
@@ -22,12 +23,26 @@ use ethjson;
 use transaction::SignedTransaction;
 use vm::EnvInfo;
 
-pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
+use super::HookType;
+
+/// Run state jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
+}
+
+/// Run state jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, json_chain_test, h)
+}
+
+pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::state::test::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		{
 			let multitransaction = test.transaction;
 			let env: EnvInfo = test.env.into();
@@ -50,7 +65,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 					let transaction: SignedTransaction = multitransaction.select(&state.indexes).into();
 
 					let result = || -> Result<_, EvmTestError> {
-						Ok(EvmTestClient::from_pod_state(spec, pre.clone())?
+						Ok(EvmTestClient::from_pod_state(&spec, pre.clone())?
 							.transact(&env, transaction, trace::NoopTracer, trace::NoopVMTracer))
 					};
 					match result() {
@@ -81,6 +96,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 			}
 		}
 
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	if !failed.is_empty() {
@@ -89,11 +105,13 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec<String> {
 	failed
 }
 
+#[cfg(test)]
 mod state_tests {
 	use super::json_chain_test;
+	use json_tests::HookType;
 
-	fn do_json_test(json_data: &[u8]) -> Vec<String> {
-		json_chain_test(json_data)
+	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+		json_chain_test(json_data, h)
 	}
 
 	declare_test!{GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"}

ethcore/src/json_tests/test_common.rs

@@ -21,7 +21,20 @@ use std::path::Path;
 use std::ffi::OsString;
 pub use ethereum_types::{H256, U256, Address};
 
-pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u8]) -> Vec<String>) {
+/// Indicate when to run the hook passed to test functions.
+#[derive(Copy, Clone, Eq, PartialEq)]
+pub enum HookType {
+	/// Hook to code to run on test start.
+	OnStart,
+	/// Hook to code to run on test end.
+	OnStop
+}
+
+pub fn run_test_path<H: FnMut(&str, HookType)>(
+	p: &Path, skip: &[&'static str],
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H
+) {
 	let path = Path::new(p);
 	let s: HashSet<OsString> = skip.iter().map(|s| {
 		let mut os: OsString = s.into();
@@ -36,33 +49,39 @@ pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u
 			} else {
 				Some(e.path())
 			}}) {
-			run_test_path(&p, skip, runner)
+			run_test_path(&p, skip, runner, start_stop_hook)
 		}
 	} else {
 		let mut path = p.to_path_buf();
 		path.set_extension("json");
-		run_test_file(&path, runner)
+		run_test_file(&path, runner, start_stop_hook)
 	}
 }
 
-pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec<String>) {
+pub fn run_test_file<H: FnMut(&str, HookType)>(
+	path: &Path,
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H
+) {
 	let mut data = Vec::new();
 	let mut file = File::open(&path).expect("Error opening test file");
 	file.read_to_end(&mut data).expect("Error reading test file");
-	let results = runner(&data);
+	let results = runner(&data, start_stop_hook);
 	let empty: [String; 0] = [];
 	assert_eq!(results, empty);
 }
 
+#[cfg(test)]
 macro_rules! test {
 	($name: expr, $skip: expr) => {
-		::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test);
+		::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test, &mut |_, _| ());
 	}
 }
 
 #[macro_export]
 macro_rules! declare_test {
 	(skip => $arr: expr, $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[test]
 		#[allow(non_snake_case)]
 		fn $id() {
@@ -70,6 +89,7 @@ macro_rules! declare_test {
 		}
 	};
 	(ignore => $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[ignore]
 		#[test]
 		#[allow(non_snake_case)]
@@ -78,6 +98,7 @@ macro_rules! declare_test {
 		}
 	};
 	(heavy => $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[cfg(feature = "test-heavy")]
 		#[test]
 		#[allow(non_snake_case)]
@@ -86,6 +107,7 @@ macro_rules! declare_test {
 		}
 	};
 	($id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[test]
 		#[allow(non_snake_case)]
 		fn $id() {

ethcore/src/json_tests/transaction.rs

@@ -14,19 +14,32 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
 use super::test_common::*;
 use evm;
 use ethjson;
 use rlp::Rlp;
 use transaction::{Action, UnverifiedTransaction, SignedTransaction};
 
-fn do_json_test(json_data: &[u8]) -> Vec<String> {
+/// Run transaction jsontests on a given folder.
+pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+}
+
+/// Run transaction jsontests on a given file.
+pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+	::json_tests::test_common::run_test_file(p, do_json_test, h)
+}
+
+fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	let tests = ethjson::transaction::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 	let frontier_schedule = evm::Schedule::new_frontier();
 	let homestead_schedule = evm::Schedule::new_homestead();
 	let byzantium_schedule = evm::Schedule::new_byzantium();
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		let mut fail_unless = |cond: bool, title: &str| if !cond { failed.push(name.clone()); println!("Transaction failed: {:?}: {:?}", name, title); };
 
 		let number: Option<u64> = test.block_number.map(Into::into);
@@ -69,6 +82,8 @@ fn do_json_test(json_data: &[u8]) -> Vec<String> {
 				Action::Create => fail_unless(None == to, "create mismatch"),
 			}
 		}
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	for f in &failed {

ethcore/src/json_tests/trie.rs

@@ -19,12 +19,21 @@ use trie::{TrieFactory, TrieSpec};
 use ethereum_types::H256;
 use memorydb::MemoryDB;
 
-fn test_trie(json: &[u8], trie: TrieSpec) -> Vec<String> {
+use super::HookType;
+
+pub use self::generic::run_test_path as run_generic_test_path;
+pub use self::generic::run_test_file as run_generic_test_file;
+pub use self::secure::run_test_path as run_secure_test_path;
+pub use self::secure::run_test_file as run_secure_test_file;
+
+fn test_trie<H: FnMut(&str, HookType)>(json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec<String> {
 	let tests = ethjson::trie::Test::load(json).unwrap();
 	let factory = TrieFactory::new(trie);
 	let mut result = vec![];
 
 	for (name, test) in tests.into_iter() {
+		start_stop_hook(&name, HookType::OnStart);
+
 		let mut memdb = MemoryDB::new();
 		let mut root = H256::default();
 		let mut t = factory.create(&mut memdb, &mut root);
@@ -39,6 +48,8 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec<String> {
 		if *t.root() != test.root.into() {
 			result.push(format!("Trie test '{:?}' failed.", name));
 		}
+
+		start_stop_hook(&name, HookType::OnStop);
 	}
 
 	for i in &result {
@@ -49,10 +60,23 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec<String> {
 }
 
 mod generic {
+	use std::path::Path;
 	use trie::TrieSpec;
 
-	fn do_json_test(json: &[u8]) -> Vec<String> {
-		super::test_trie(json, TrieSpec::Generic)
+	use super::HookType;
+
+	/// Run generic trie jsontests on a given folder.
+	pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+		::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+	}
+
+	/// Run generic trie jsontests on a given file.
+	pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+		::json_tests::test_common::run_test_file(p, do_json_test, h)
+	}
+
+	fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
+		super::test_trie(json, TrieSpec::Generic, h)
 	}
 
 	declare_test!{TrieTests_trietest, "TrieTests/trietest"}
@@ -60,10 +84,23 @@ mod generic {
 }
 
 mod secure {
+	use std::path::Path;
 	use trie::TrieSpec;
 
-	fn do_json_test(json: &[u8]) -> Vec<String> {
-		super::test_trie(json, TrieSpec::Secure)
+	use super::HookType;
+
+	/// Run secure trie jsontests on a given folder.
+	pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+		::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+	}
+
+	/// Run secure trie jsontests on a given file.
+	pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+		::json_tests::test_common::run_test_file(p, do_json_test, h)
+	}
+
+	fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
+		super::test_trie(json, TrieSpec::Secure, h)
 	}
 
 	declare_test!{TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"}

ethcore/src/lib.rs

@@ -109,6 +109,7 @@ extern crate vm;
 extern crate wasm;
 extern crate memory_cache;
 extern crate journaldb;
+#[cfg(any(test, feature = "json-tests", feature = "test-helpers"))]
 extern crate tempdir;
 
 #[macro_use]
@@ -168,9 +169,8 @@ mod tx_filter;
 
 #[cfg(test)]
 mod tests;
-#[cfg(test)]
 #[cfg(feature = "json-tests")]
-mod json_tests;
+pub mod json_tests;
 #[cfg(any(test, feature = "test-helpers"))]
 pub mod test_helpers;
 

evmbin/Cargo.toml

@@ -10,7 +10,7 @@ path = "./src/main.rs"
 
 [dependencies]
 docopt = "0.8"
-ethcore = { path = "../ethcore", features = ["test-helpers"] }
+ethcore = { path = "../ethcore", features = ["test-helpers", "json-tests"] }
 ethjson = { path = "../json" }
 ethcore-bytes = { path = "../util/bytes" }
 ethcore-transaction = { path = "../ethcore/transaction" }

evmbin/src/info.rs

@@ -105,7 +105,7 @@ pub fn run_transaction<T: Informant>(
 
 	informant.set_gas(env_info.gas_limit);
 
-	let result = run(spec, env_info.gas_limit, pre_state, |mut client| {
+	let result = run(&spec, env_info.gas_limit, pre_state, |mut client| {
 		let result = client.transact(env_info, transaction, trace::NoopTracer, informant);
 		match result {
 			TransactResult::Ok { state_root, .. } if state_root != post_root => {

evmbin/src/main.rs

@@ -46,7 +46,7 @@ use docopt::Docopt;
 use rustc_hex::FromHex;
 use ethereum_types::{U256, Address};
 use bytes::Bytes;
-use ethcore::spec;
+use ethcore::{spec, json_tests};
 use vm::{ActionParams, CallType};
 
 mod info;
@@ -61,9 +61,16 @@ EVM implementation for Parity.
 Usage:
     parity-evm state-test <file> [--json --std-json --only NAME --chain CHAIN]
     parity-evm stats [options]
+    parity-evm stats-jsontests-vm <file>
     parity-evm [options]
     parity-evm [-h | --help]
 
+Commands:
+    state-test         Run a state test from a json file.
+    stats              Execute EVM runtime code and return the statistics.
+    stats-jsontests-vm Execute standard jsontests format VMTests and return
+                       timing statistcis in tsv format.
+
 Transaction options:
     --code CODE        Contract code as hex (without 0x).
     --to ADDRESS       Recipient address (without 0x).
@@ -90,6 +97,8 @@ fn main() {
 
 	if args.cmd_state_test {
 		run_state_test(args)
+	} else if args.cmd_stats_jsontests_vm {
+		run_stats_jsontests_vm(args)
 	} else if args.flag_json {
 		run_call(args, display::json::Informant::default())
 	} else if args.flag_std_json {
@@ -99,6 +108,40 @@ fn main() {
 	}
 }
 
+fn run_stats_jsontests_vm(args: Args) {
+	use json_tests::HookType;
+	use std::collections::HashMap;
+	use std::time::{Instant, Duration};
+
+	let file = args.arg_file.expect("FILE (or PATH) is required");
+
+	let mut timings: HashMap<String, (Instant, Option<Duration>)> = HashMap::new();
+
+	{
+		let mut record_time = |name: &str, typ: HookType| {
+			match typ {
+				HookType::OnStart => {
+					timings.insert(name.to_string(), (Instant::now(), None));
+				},
+				HookType::OnStop => {
+					timings.entry(name.to_string()).and_modify(|v| {
+						v.1 = Some(v.0.elapsed());
+					});
+				},
+			}
+		};
+		if !file.is_file() {
+			json_tests::run_executive_test_path(&file, &[], &mut record_time);
+		} else {
+			json_tests::run_executive_test_file(&file, &mut record_time);
+		}
+	}
+
+	for (name, v) in timings {
+		println!("{}\t{}", name, display::as_micros(&v.1.expect("All hooks are called with OnStop; qed")));
+	}
+}
+
 fn run_state_test(args: Args) {
 	use ethjson::state::test::Test;
 
@@ -179,6 +222,7 @@ fn run_call<T: Informant>(args: Args, informant: T) {
 struct Args {
 	cmd_stats: bool,
 	cmd_state_test: bool,
+	cmd_stats_jsontests_vm: bool,
 	arg_file: Option<PathBuf>,
 	flag_only: Option<String>,
 	flag_from: Option<String>,

scripts/aura-test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e # fail on any error
 set -u # treat unset variables as error
 

scripts/cov.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Installing KCOV under ubuntu
 # https://users.rust-lang.org/t/tutorial-how-to-collect-test-coverages-for-rust-project/650#
 ### Install deps

scripts/doc.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 # generate documentation only for partiy and ethcore libraries
 
 cargo doc --no-deps --verbose --all --exclude parity-ipfs-api &&

scripts/docker-build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 cd docker/hub
 DOCKER_BUILD_TAG=$1
 echo "Docker build tag: " $DOCKER_BUILD_TAG

scripts/evm_jsontests_bench.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+cargo build --release -p evmbin
+
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmArithmeticTest
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBitwiseLogicOperation
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBlockInfoTest
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmEnvironmentalInfo
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmIOandFlowOperations
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmLogTest
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPerformance
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPushDupSwapTest
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmRandomTest
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSha3Test
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSystemOperations
+./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmTests

scripts/evm_uint_bench.sh

@@ -1,23 +1,25 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-set -e
-
-cargo build --release
+cargo build --release -p evmbin
 
 # LOOP TEST
 CODE1=606060405260005b620f42408112156019575b6001016007565b600081905550600680602b6000396000f3606060405200
-ethvm --code $CODE1
-echo "^^^^ ethvm"
-./target/release/evm stats --code $CODE1 --gas 4402000
+if [ -x "$(command -v ethvm)" ]; then
+  ethvm --code $CODE1
+  echo "^^^^ ethvm"
+fi
+./target/release/parity-evm stats --code $CODE1 --gas 4402000
 echo "^^^^ usize"
-./target/release/evm stats --code $CODE1
+./target/release/parity-evm stats --code $CODE1
 echo "^^^^ U256"
 
 # RNG TEST
 CODE2=6060604052600360056007600b60005b620f4240811215607f5767ffe7649d5eca84179490940267f47ed85c4b9a6379019367f8e5dd9a5c994bba9390930267f91d87e4b8b74e55019267ff97f6f3b29cda529290920267f393ada8dd75c938019167fe8d437c45bb3735830267f47d9a7b5428ffec019150600101600f565b838518831882186000555050505050600680609a6000396000f3606060405200
-ethvm --code $CODE2
-echo "^^^^ ethvm"
-./target/release/evm stats --code $CODE2 --gas 143020115
+if [ -x "$(command -v ethvm)" ]; then
+  ethvm --code $CODE2
+  echo "^^^^ ethvm"
+fi
+./target/release/parity-evm stats --code $CODE2 --gas 143020115
 echo "^^^^ usize"
-./target/release/evm stats --code $CODE2
+./target/release/parity-evm stats --code $CODE2
 echo "^^^^ U256"

scripts/gitlab-build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e # fail on any error
 set -u # treat unset variables as error

scripts/gitlab-push-release.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e # fail on any error
 set -u # treat unset variables as error

scripts/gitlab-test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #ARGUMENT test for RUST and COVERAGE
 set -e # fail on any error
 set -u # treat unset variables as error

scripts/hook.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 FILE=./.git/hooks/pre-push
 
 echo "#!/bin/sh\n" > $FILE

scripts/safe_curl.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -eu
 
@@ -17,4 +17,3 @@ else
 	echo 'Unable to push info to updater service.';
 	exit 2
 fi
-

scripts/validate_chainspecs.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
 ERR=0
 cargo build --release -p chainspec
@@ -12,4 +12,3 @@ for spec in ethcore/res/ethereum/*.json; do
 done
 
 exit $ERR
-