Running state test using parity-evm (#6355)

* Initial version of state tests.

* Refactor state to support tracing.

* Unify TransactResult.

* Add test.

evmbin/Cargo.toml

@@ -14,6 +14,7 @@ docopt = "0.8"
 serde = "1.0"
 serde_derive = "1.0"
 ethcore = { path = "../ethcore" }
+ethjson = { path = "../json" }
 ethcore-util = { path = "../util" }
 evm = { path = "../ethcore/evm" }
 vm = { path = "../ethcore/vm" }

evmbin/src/display/json.rs

@@ -30,10 +30,8 @@ pub struct Informant {
 	depth: usize,
 	pc: usize,
 	instruction: u8,
-	name: &'static str,
 	gas_cost: U256,
 	gas_used: U256,
-	stack_pop: usize,
 	stack: Vec<U256>,
 	memory: Vec<u8>,
 	storage: HashMap<H256, H256>,
@@ -58,11 +56,19 @@ impl Informant {
 }
 
 impl vm::Informant for Informant {
+	fn before_test(&self, name: &str, action: &str) {
+		println!(
+			"{{\"test\":\"{name}\",\"action\":\"{action}\"}}",
+			name = name,
+			action = action,
+		);
+	}
+
 	fn set_gas(&mut self, gas: U256) {
 		self.gas_used = gas;
 	}
 
-	fn finish(&mut self, result: Result<vm::Success, vm::Failure>) {
+	fn finish(result: Result<vm::Success, vm::Failure>) {
 		match result {
 			Ok(success) => println!(
 				"{{\"output\":\"0x{output}\",\"gasUsed\":\"{gas:x}\",\"time\":{time}}}",
@@ -112,7 +118,7 @@ impl trace::VMTracer for Informant {
 		self.gas_used = gas_used;
 
 		let len = self.stack.len();
-		self.stack.truncate(len - info.args);
+		self.stack.truncate(if len > info.args { len - info.args } else { 0 });
 		self.stack.extend_from_slice(stack_push);
 
 		if let Some((pos, data)) = mem_diff {

evmbin/src/display/simple.rs

@@ -27,7 +27,11 @@ use info as vm;
 pub struct Informant;
 
 impl vm::Informant for Informant {
-	fn finish(&mut self, result: Result<vm::Success, vm::Failure>) {
+	fn before_test(&self, name: &str, action: &str) {
+		println!("Test: {} ({})", name, action);
+	}
+
+	fn finish(result: Result<vm::Success, vm::Failure>) {
 		match result {
 			Ok(success) => {
 				println!("Output: 0x{}", success.output.to_hex());

evmbin/src/info.rs

@@ -17,17 +17,19 @@
 //! VM runner.
 
 use std::time::{Instant, Duration};
-use util::U256;
-use ethcore::{trace, spec};
-use ethcore::client::{EvmTestClient, EvmTestError};
-use vm::ActionParams;
+use util::{U256, H256};
+use ethcore::{trace, spec, transaction, pod_state};
+use ethcore::client::{self, EvmTestClient, EvmTestError, TransactResult};
+use ethjson;
 
 /// VM execution informant
 pub trait Informant: trace::VMTracer {
+	/// Display a single run init message
+	fn before_test(&self, test: &str, action: &str);
 	/// Set initial gas.
 	fn set_gas(&mut self, _gas: U256) {}
 	/// Display final result.
-	fn finish(&mut self, result: Result<Success, Failure>);
+	fn finish(result: Result<Success, Failure>);
 }
 
 /// Execution finished correctly
@@ -50,17 +52,71 @@ pub struct Failure {
 	pub time: Duration,
 }
 
+/// Execute given Transaction and verify resulting state root.
+pub fn run_transaction<T: Informant>(
+	name: &str,
+	idx: usize,
+	spec: &ethjson::state::test::ForkSpec,
+	pre_state: &pod_state::PodState,
+	post_root: H256,
+	env_info: &client::EnvInfo,
+	transaction: transaction::SignedTransaction,
+	mut informant: T,
+) {
+	let spec_name = format!("{:?}", spec).to_lowercase();
+	let spec = match EvmTestClient::spec_from_json(spec) {
+		Some(spec) => {
+			informant.before_test(&format!("{}:{}:{}", name, spec_name, idx), "starting");
+			spec
+		},
+		None => {
+			informant.before_test(&format!("{}:{}:{}", name, spec_name, idx), "skipping because of missing spec");
+			return;
+		},
+	};
+
+	informant.set_gas(env_info.gas_limit);
+
+	let result = run(spec, env_info.gas_limit, pre_state, |mut client| {
+		let result = client.transact(env_info, transaction, informant);
+		match result {
+			TransactResult::Ok { state_root, .. } if state_root != post_root => {
+				Err(EvmTestError::PostCondition(format!(
+					"State root mismatch (got: {}, expected: {})",
+					state_root,
+					post_root,
+				)))
+			},
+			TransactResult::Ok { gas_left, output, .. } => {
+				Ok((gas_left, output))
+			},
+			TransactResult::Err { error, .. } => {
+				Err(EvmTestError::PostCondition(format!(
+					"Unexpected execution error: {:?}", error
+				)))
+			},
+		}
+	});
+
+	T::finish(result)
+}
+
 /// Execute VM with given `ActionParams`
-pub fn run<T: trace::VMTracer>(vm_tracer: &mut T, spec: spec::Spec, params: ActionParams) -> Result<Success, Failure> {
-	let mut test_client = EvmTestClient::new(spec).map_err(|error| Failure {
+pub fn run<'a, F, T>(spec: &'a spec::Spec, initial_gas: U256, pre_state: T, run: F) -> Result<Success, Failure> where
+	F: FnOnce(EvmTestClient) -> Result<(U256, Vec<u8>), EvmTestError>,
+	T: Into<Option<&'a pod_state::PodState>>,
+{
+	let test_client = match pre_state.into() {
+		Some(pre_state) => EvmTestClient::from_pod_state(spec, pre_state.clone()),
+		None => EvmTestClient::new(spec),
+	}.map_err(|error| Failure {
 		gas_used: 0.into(),
 		error,
 		time: Duration::from_secs(0)
 	})?;
 
-	let initial_gas = params.gas;
 	let start = Instant::now();
-	let result = test_client.call(params, vm_tracer);
+	let result = run(test_client);
 	let duration = start.elapsed();
 
 	match result {

evmbin/src/main.rs

@@ -17,8 +17,9 @@
 //! Parity EVM interpreter binary.
 
 #![warn(missing_docs)]
-#![allow(dead_code)]
+
 extern crate ethcore;
+extern crate ethjson;
 extern crate rustc_hex;
 extern crate serde;
 #[macro_use]
@@ -31,6 +32,7 @@ extern crate panic_hook;
 
 use std::sync::Arc;
 use std::{fmt, fs};
+use std::path::PathBuf;
 use docopt::Docopt;
 use rustc_hex::FromHex;
 use util::{U256, Bytes, Address};
@@ -47,6 +49,7 @@ EVM implementation for Parity.
   Copyright 2016, 2017 Parity Technologies (UK) Ltd
 
 Usage:
+    parity-evm state-test <file> [--json --only NAME --chain CHAIN]
     parity-evm stats [options]
     parity-evm [options]
     parity-evm [-h | --help]
@@ -59,6 +62,10 @@ Transaction options:
     --gas GAS          Supplied gas as hex (without 0x).
     --gas-price WEI    Supplied gas price as hex (without 0x).
 
+State test options:
+    --only NAME        Runs only a single test matching the name.
+    --chain CHAIN      Run only tests from specific chain.
+
 General options:
     --json             Display verbose results in JSON.
     --chain CHAIN      Chain spec file path.
@@ -71,20 +78,67 @@ fn main() {
 
 	let args: Args = Docopt::new(USAGE).and_then(|d| d.deserialize()).unwrap_or_else(|e| e.exit());
 
-	if args.flag_json {
-		run(args, display::json::Informant::default())
+	if args.cmd_state_test {
+		run_state_test(args)
+	} else if args.flag_json {
+		run_call(args, display::json::Informant::default())
 	} else {
-		run(args, display::simple::Informant::default())
+		run_call(args, display::simple::Informant::default())
 	}
 }
 
-fn run<T: Informant>(args: Args, mut informant: T) {
+fn run_state_test(args: Args) {
+	use ethjson::state::test::Test;
+
+	let file = args.arg_file.expect("FILE is required");
+	let mut file = match fs::File::open(&file) {
+		Err(err) => die(format!("Unable to open: {:?}: {}", file, err)),
+		Ok(file) => file,
+	};
+	let state_test = match Test::load(&mut file) {
+		Err(err) => die(format!("Unable to load the test file: {}", err)),
+		Ok(test) => test,
+	};
+	let only_test = args.flag_only.map(|s| s.to_lowercase());
+	let only_chain = args.flag_chain.map(|s| s.to_lowercase());
+
+	for (name, test) in state_test {
+		if let Some(false) = only_test.as_ref().map(|only_test| &name.to_lowercase() == only_test) {
+			continue;
+		}
+
+		let multitransaction = test.transaction;
+		let env_info = test.env.into();
+		let pre = test.pre_state.into();
+
+		for (spec, states) in test.post_states {
+			if let Some(false) = only_chain.as_ref().map(|only_chain| &format!("{:?}", spec).to_lowercase() == only_chain) {
+				continue;
+			}
+
+			for (idx, state) in states.into_iter().enumerate() {
+				let post_root = state.hash.into();
+				let transaction = multitransaction.select(&state.indexes).into();
+
+				if args.flag_json {
+					let i = display::json::Informant::default();
+					info::run_transaction(&name, idx, &spec, &pre, post_root, &env_info, transaction, i)
+				} else {
+					let i = display::simple::Informant::default();
+					info::run_transaction(&name, idx, &spec, &pre, post_root, &env_info, transaction, i)
+				}
+			}
+		}
+	}
+}
+
+fn run_call<T: Informant>(args: Args, mut informant: T) {
 	let from = arg(args.from(), "--from");
 	let to = arg(args.to(), "--to");
 	let code = arg(args.code(), "--code");
 	let spec = arg(args.spec(), "--chain");
 	let gas = arg(args.gas(), "--gas");
-	let gas_price = arg(args.gas(), "--gas-price");
+	let gas_price = arg(args.gas_price(), "--gas-price");
 	let data = arg(args.data(), "--input");
 
 	if code.is_none() && to == Address::default() {
@@ -103,13 +157,18 @@ fn run<T: Informant>(args: Args, mut informant: T) {
 	params.data = data;
 
 	informant.set_gas(gas);
-	let result = info::run(&mut informant, spec, params);
-	informant.finish(result);
+	let result = info::run(&spec, gas, None, |mut client| {
+		client.call(params, &mut informant)
+	});
+	T::finish(result);
 }
 
 #[derive(Debug, Deserialize)]
 struct Args {
 	cmd_stats: bool,
+	cmd_state_test: bool,
+	arg_file: Option<PathBuf>,
+	flag_only: Option<String>,
 	flag_from: Option<String>,
 	flag_to: Option<String>,
 	flag_code: Option<String>,
@@ -221,4 +280,22 @@ mod tests {
 		assert_eq!(args.data(), Ok(Some(vec![06])));
 		assert_eq!(args.flag_chain, Some("./testfile".to_owned()));
 	}
+
+	#[test]
+	fn should_parse_state_test_command() {
+		let args = run(&[
+			"parity-evm",
+			"state-test",
+			"./file.json",
+			"--chain", "homestead",
+			"--only=add11",
+			"--json",
+		]);
+
+		assert_eq!(args.cmd_state_test, true);
+		assert!(args.arg_file.is_some());
+		assert_eq!(args.flag_json, true);
+		assert_eq!(args.flag_chain, Some("homestead".to_owned()));
+		assert_eq!(args.flag_only, Some("add11".to_owned()));
+	}
 }