EVM benchmark utilities (#8944)

* Make it possible to expose jsontests using feature flag

* Add start_stop_hook for jsontests

* Fix evmbin compile

* Implement vm jsontests times to tsv result

* Use /usr/bin/env to avoid errors on non-Debian systems

* Move evmbin/bench.sh to scripts and add vm_jsontests script for convenience

* Add tempdir as required deps for test-helpers

* Address grumbles on comments

* Detect file/folder automatically and add command docs

* Fix bench script

* times -> timings

* typo: wrong if condition

ethcore/src/json_tests/test_common.rs

@@ -21,7 +21,20 @@ use std::path::Path;
 use std::ffi::OsString;
 pub use ethereum_types::{H256, U256, Address};
 
-pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u8]) -> Vec<String>) {
+/// Indicate when to run the hook passed to test functions.
+#[derive(Copy, Clone, Eq, PartialEq)]
+pub enum HookType {
+	/// Hook to code to run on test start.
+	OnStart,
+	/// Hook to code to run on test end.
+	OnStop
+}
+
+pub fn run_test_path<H: FnMut(&str, HookType)>(
+	p: &Path, skip: &[&'static str],
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H
+) {
 	let path = Path::new(p);
 	let s: HashSet<OsString> = skip.iter().map(|s| {
 		let mut os: OsString = s.into();
@@ -36,33 +49,39 @@ pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u
 			} else {
 				Some(e.path())
 			}}) {
-			run_test_path(&p, skip, runner)
+			run_test_path(&p, skip, runner, start_stop_hook)
 		}
 	} else {
 		let mut path = p.to_path_buf();
 		path.set_extension("json");
-		run_test_file(&path, runner)
+		run_test_file(&path, runner, start_stop_hook)
 	}
 }
 
-pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec<String>) {
+pub fn run_test_file<H: FnMut(&str, HookType)>(
+	path: &Path,
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H
+) {
 	let mut data = Vec::new();
 	let mut file = File::open(&path).expect("Error opening test file");
 	file.read_to_end(&mut data).expect("Error reading test file");
-	let results = runner(&data);
+	let results = runner(&data, start_stop_hook);
 	let empty: [String; 0] = [];
 	assert_eq!(results, empty);
 }
 
+#[cfg(test)]
 macro_rules! test {
 	($name: expr, $skip: expr) => {
-		::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test);
+		::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test, &mut |_, _| ());
 	}
 }
 
 #[macro_export]
 macro_rules! declare_test {
 	(skip => $arr: expr, $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[test]
 		#[allow(non_snake_case)]
 		fn $id() {
@@ -70,6 +89,7 @@ macro_rules! declare_test {
 		}
 	};
 	(ignore => $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[ignore]
 		#[test]
 		#[allow(non_snake_case)]
@@ -78,6 +98,7 @@ macro_rules! declare_test {
 		}
 	};
 	(heavy => $id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[cfg(feature = "test-heavy")]
 		#[test]
 		#[allow(non_snake_case)]
@@ -86,6 +107,7 @@ macro_rules! declare_test {
 		}
 	};
 	($id: ident, $name: expr) => {
+		#[cfg(test)]
 		#[test]
 		#[allow(non_snake_case)]
 		fn $id() {