Update JSON tests to d4f86ecf4aa7c (#11054)

* new ethereum consensus tests, #10908

* Update JSON tests to 725dbc73a

This PR reverts the controversial changes of the previous PR and skips the failing tests.

Maybe I misunderstand the suggested workaround of putting the fix under `#[cfg(test)]` but it seems odd to run different code in production than we run in tests. Instead here I suggest we skip the failing tests with the argument that we do not wish to fix this issue (at least not at this time) because it does not affect us. If I am wrong, and I likely am, I look forward to hearing why and what a better approach to updating the state tests is.

Branched off https://github.com/paritytech/parity-ethereum/pull/10923

ref #10908

* Update json test commit to 1dc9d20e97165708f7db0bbf2d1a87a6b4285827

* Fail with error message

* Handle missing r, s, v params in json tests
Light cleanup of json test runner

* Include the path to the test file

* Handle new `postState` format: string or map
Sort out tests
Missing docs

* WIP

* Include test-helpers from ethjson

* Sort out new paths

* Remove dead code

* Fix warnings stemming from code called only from macros
Skip failing tests in stRevert/ and stTransactionTest/ (too course a filter!)
Docs and light touch refactorings for readability

* Skip all failing tests

* Document the single-test-skipping madness

* Update tests to latest commit on the `develop` branch

* Rename test skipping types to reflect actual purpose

* Switch to skipping individual tests in currents.json
Add some logging to help debug skipping

* Fix rpc test by curve fitting to new json test source file

* Add refs to all issues for fixing failing&skipped json tests

* Sort out the need for Clone for tests

* [json-tests] populate state from genesis pod state (#11083)

* [json-tests] populate state from genesis pod state

* [json-tests] #11075 is resolved as well

* [json-tests] #11076 hopefully too

* [json-tests] #11077 🎉

* [json-tests] fix trailing comma

* Update ethcore/src/json_tests/chain.rs

Co-Authored-By: Andronik Ordian <write@reusable.software>

* Add issue numbers to TODOs

* Apply @ordians fix for wrong state_root

* Warn on invalid RLP

* Remove the `ci-skip-tests` feature

Cargo.toml

@@ -93,7 +93,6 @@ default = ["accounts"]
 accounts = ["ethcore-accounts", "parity-rpc/accounts"]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
-ci-skip-tests = ["ethcore/ci-skip-tests"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethcore/Cargo.toml

@@ -118,15 +118,13 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 slow-blocks = []
 # Run JSON consensus tests.
 json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers"]
-# Skip JSON consensus tests with pending issues.
-ci-skip-tests = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile test helpers
 # note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works
 test-helpers = [
     "blooms-db",
-    "ethjson",
+    "ethjson/test-helpers",
     "ethkey",
     "kvdb-memorydb",
     "kvdb-rocksdb",

ethcore/account-state/src/state.rs

@@ -746,7 +746,7 @@ impl<B: Backend> State<B> {
 	/// Propagate local cache into shared canonical state cache.
 	fn propagate_to_global_cache(&mut self) {
 		let mut addresses = self.cache.borrow_mut();
-		trace!("Committing cache {:?} entries", addresses.len());
+		trace!(target: "state", "Committing cache {:?} entries", addresses.len());
 		for (address, a) in addresses.drain().filter(|&(_, ref a)| a.state == AccountState::Committed || a.state == AccountState::CleanFresh) {
 			self.db.add_to_account_cache(address, a.account, a.state == AccountState::Committed);
 		}

ethcore/res/ethereum/tests

@@ -1 +1 @@
-Subproject commit 725dbc73a54649e22a00330bd0f4d6699a5060e5
+Subproject commit d4f86ecf4aa7c44a40bc0c972fd3e25d63ef5d92

ethcore/res/ethereum/tests-issues/currents.json

@@ -1,4 +1,57 @@
 {
-	"block": [],
+	"block": [
-	"state": []
+		{
+			"reference": "Issue https://github.com/paritytech/parity-ethereum/issues/11073 (also see https://github.com/paritytech/parity-ethereum/pull/10923)",
+			"failing": "stRevertTest",
+			"subtests": [
+				"RevertPrecompiledTouch_d0g0v0_Byzantium",
+				"RevertPrecompiledTouch_d0g0v0_Constantinople",
+				"RevertPrecompiledTouch_d0g0v0_ConstantinopleFix",
+				"RevertPrecompiledTouch_d0g0v0_EIP158",
+				"RevertPrecompiledTouch_d3g0v0_ConstantinopleFix",
+				"RevertPrecompiledTouchCC_d0g0v0_Byzantium",
+				"RevertPrecompiledTouchCC_d0g0v0_Constantinople",
+				"RevertPrecompiledTouchCC_d0g0v0_EIP158",
+				"RevertPrecompiledTouchDC_d0g0v0_Byzantium",
+				"RevertPrecompiledTouchDC_d0g0v0_Constantinople",
+				"RevertPrecompiledTouchDC_d0g0v0_EIP158",
+				"RevertPrecompiledTouchExactOOG_d7g1v0_ConstantinopleFix",
+				"RevertPrecompiledTouchExactOOG_d31g1v0_ConstantinopleFix",
+				"RevertPrecompiledTouch_storage_d3g0v0_ConstantinopleFix",
+				"RevertPrecompiledTouch_storage_d0g0v0_ConstantinopleFix"
+			]
+		}
+	],
+	"state": [
+		{
+			"reference": "Issue https://github.com/paritytech/parity-ethereum/issues/11078 (also see https://github.com/paritytech/parity-ethereum/pull/10923)",
+			"failing": "stRevertTest",
+			"subtests": {
+				"RevertPrecompiledTouch_storage": {
+					"subnumbers": ["1", "2"],
+					"chain": "St. Peter's (test)"
+				}
+			}
+		},
+		{
+			"reference": "Issue https://github.com/paritytech/parity-ethereum/issues/11079 (also see https://github.com/paritytech/parity-ethereum/pull/10923)",
+			"failing": "stRevertTest",
+			"subtests": {
+				"RevertPrecompiledTouchExactOOG": {
+					"subnumbers": ["61", "64"],
+					"chain": "St. Peter's (test)"
+				}
+			}
+		},
+		{
+			"reference": "Issue https://github.com/paritytech/parity-ethereum/issues/11080 (also see https://github.com/paritytech/parity-ethereum/pull/10923)",
+			"failing": "stRevertTest",
+			"subtests": {
+				"RevertPrecompiledTouch": {
+					"subnumbers": ["1", "2"],
+					"chain": "St. Peter's (test)"
+				}
+			}
+		}
+	]
 }

ethcore/spec/src/genesis.rs

@@ -20,6 +20,7 @@ use keccak_hash::KECCAK_NULL_RLP;
 use crate::seal::Seal;
 
 /// Genesis components.
+#[derive(Debug)]
 pub struct Genesis {
 	/// Seal.
 	pub seal: Seal,

ethcore/spec/src/seal.rs

@@ -21,6 +21,7 @@ use ethereum_types::{H64, H256, H520};
 use ethjson;
 
 /// Classic ethereum seal.
+#[derive(Debug)]
 pub struct Ethereum {
 	/// Seal nonce.
 	pub nonce: H64,
@@ -37,6 +38,7 @@ impl Into<Generic> for Ethereum {
 }
 
 /// AuthorityRound seal.
+#[derive(Debug)]
 pub struct AuthorityRound {
 	/// Seal step.
 	pub step: usize,
@@ -45,6 +47,7 @@ pub struct AuthorityRound {
 }
 
 /// Tendermint seal.
+#[derive(Debug)]
 pub struct Tendermint {
 	/// Seal round.
 	pub round: usize,
@@ -73,9 +76,11 @@ impl Into<Generic> for Tendermint {
 	}
 }
 
+#[derive(Debug)]
 pub struct Generic(pub Vec<u8>);
 
 /// Genesis seal type.
+#[derive(Debug)]
 pub enum Seal {
 	/// Classic ethereum seal.
 	Ethereum(Ethereum),

ethcore/spec/src/spec.rs

@@ -58,9 +58,6 @@ use crate::{
 	seal::Generic as GenericSeal,
 };
 
-
-
-
 /// Runtime parameters for the spec that are related to how the software should run the chain,
 /// rather than integral properties of the chain itself.
 pub struct SpecParams<'a> {
@@ -134,53 +131,56 @@ fn run_constructors<T: Backend>(
 	let start_nonce = engine.account_start_nonce(0);
 
 	let mut state = State::from_existing(db, root, start_nonce, factories.clone())?;
-
+	if constructors.is_empty() {
-	// Execute contract constructors.
+		state.populate_from(genesis_state.clone());
-	let env_info = EnvInfo {
+		let _ = state.commit()?;
-		number: 0,
+	} else {
-		author,
+		// Execute contract constructors.
-		timestamp,
+		let env_info = EnvInfo {
-		difficulty,
+			number: 0,
-		last_hashes: Default::default(),
+			author,
-		gas_used: U256::zero(),
+			timestamp,
-		gas_limit: U256::max_value(),
+			difficulty,
-	};
+			last_hashes: Default::default(),
-
+			gas_used: U256::zero(),
-	let from = Address::zero();
+			gas_limit: U256::max_value(),
-	for &(ref address, ref constructor) in constructors.iter() {
-		trace!(target: "spec", "run_constructors: Creating a contract at {}.", address);
-		trace!(target: "spec", "  .. root before = {}", state.root());
-		let params = ActionParams {
-			code_address: address.clone(),
-			code_hash: Some(keccak(constructor)),
-			code_version: U256::zero(),
-			address: address.clone(),
-			sender: from.clone(),
-			origin: from.clone(),
-			gas: U256::max_value(),
-			gas_price: Default::default(),
-			value: ActionValue::Transfer(Default::default()),
-			code: Some(Arc::new(constructor.clone())),
-			data: None,
-			call_type: CallType::None,
-			params_type: ParamsType::Embedded,
 		};
 
-		let mut substate = Substate::new();
+		let from = Address::zero();
+		for &(ref address, ref constructor) in constructors.iter() {
+			trace!(target: "spec", "run_constructors: Creating a contract at {}.", address);
+			trace!(target: "spec", "  .. root before = {}", state.root());
+			let params = ActionParams {
+				code_address: address.clone(),
+				code_hash: Some(keccak(constructor)),
+				code_version: U256::zero(),
+				address: address.clone(),
+				sender: from.clone(),
+				origin: from.clone(),
+				gas: U256::max_value(),
+				gas_price: Default::default(),
+				value: ActionValue::Transfer(Default::default()),
+				code: Some(Arc::new(constructor.clone())),
+				data: None,
+				call_type: CallType::None,
+				params_type: ParamsType::Embedded,
+			};
 
-		{
+			let mut substate = Substate::new();
-			let machine = engine.machine();
+
-			let schedule = machine.schedule(env_info.number);
+			{
-			let mut exec = Executive::new(&mut state, &env_info, &machine, &schedule);
+				let machine = engine.machine();
-			// failing create is not a bug
+				let schedule = machine.schedule(env_info.number);
-			if let Err(e) = exec.create(params, &mut substate, &mut NoopTracer, &mut NoopVMTracer) {
+				let mut exec = Executive::new(&mut state, &env_info, &machine, &schedule);
-				warn!(target: "spec", "Genesis constructor execution at {} failed: {}.", address, e);
+				// failing create is not a bug
+				if let Err(e) = exec.create(params, &mut substate, &mut NoopTracer, &mut NoopVMTracer) {
+					warn!(target: "spec", "Genesis constructor execution at {} failed: {}.", address, e);
+				}
 			}
+
+			let _ = state.commit()?;
 		}
-
-		let _ = state.commit()?;
 	}
-
 	Ok(state.drop())
 }
 
@@ -219,7 +219,7 @@ pub struct Spec {
 	pub hardcoded_sync: Option<SpecHardcodedSync>,
 	/// Contract constructors to be executed on genesis.
 	pub constructors: Vec<(Address, Bytes)>,
-	/// May be prepopulated if we know this in advance.
+	/// May be pre-populated if we know this in advance.
 	pub state_root: H256,
 	/// Genesis state as plain old data.
 	pub genesis_state: PodState,

ethcore/src/json_tests/chain.rs

@@ -19,49 +19,50 @@ use std::sync::Arc;
 use client::{Client, ClientConfig};
 use client_traits::{ImportBlock, ChainInfo};
 use spec::Genesis;
-use ethjson;
+use ethjson::test_helpers::blockchain;
 use miner::Miner;
 use io::IoChannel;
 use test_helpers::{self, EvmTestClient};
 use types::verification::Unverified;
-use verification::VerifierType;
+use verification::{VerifierType, queue::kind::BlockLike};
-use super::SKIP_TEST_STATE;
+use super::SKIP_TESTS;
 use super::HookType;
 
-/// Run chain jsontests on a given folder.
+#[allow(dead_code)]
-pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
-	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
-}
-
-/// Run chain jsontests on a given file.
-pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
-	::json_tests::test_common::run_test_file(p, json_chain_test, h)
-}
-
 fn skip_test(name: &String) -> bool {
-	SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name))
+	SKIP_TESTS
+		.block
+		.iter()
+		.any(|block_test|block_test.subtests.contains(name))
 }
 
-pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
+#[allow(dead_code)]
+pub fn json_chain_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	let _ = ::env_logger::try_init();
-	let tests = ethjson::test_helpers::blockchain::Test::load(json_data).unwrap();
+	let tests = blockchain::Test::load(json_data)
+		.expect(&format!("Could not parse JSON chain test data from {}", path.display()));
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
 		if skip_test(&name) {
-			println!("   - {} | {:?} Ignoring tests because in skip list", name, blockchain.network);
+			println!("   - {} | {:?}: SKIPPED", name, blockchain.network);
 			continue;
 		}
+
 		start_stop_hook(&name, HookType::OnStart);
 
 		let mut fail = false;
 		{
-			let mut fail_unless = |cond: bool| if !cond && !fail {
+			let mut fail_unless = |cond: bool| {
-				failed.push(name.clone());
+				if !cond && !fail {
-				flushln!("FAIL");
+					failed.push(name.clone());
-				fail = true;
+					flushln!("FAIL");
-				true
+					fail = true;
-			} else {false};
+					true
+				} else {
+					false
+				}
+			};
 
 			flush!("   - {}...", name);
 
@@ -69,7 +70,7 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 				let mut spec = match EvmTestClient::fork_spec_from_json(&blockchain.network) {
 					Some(spec) => spec,
 					None => {
-						println!("   - {} | {:?} Ignoring tests because of missing spec", name, blockchain.network);
+						println!("   - {} | {:?} Ignoring tests because of missing chainspec", name, blockchain.network);
 						continue;
 					}
 				};
@@ -89,17 +90,32 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 					config.check_seal = false;
 				}
 				config.history = 8;
+				config.queue.verifier_settings.num_verifiers = 1;
 				let client = Client::new(
 					config,
 					&spec,
 					db,
 					Arc::new(Miner::new_for_tests(&spec, None)),
 					IoChannel::disconnected(),
-				).unwrap();
+				).expect("Failed to instantiate a new Client");
+
 				for b in blockchain.blocks_rlp() {
-					if let Ok(block) = Unverified::from_rlp(b) {
+					let bytes_len = b.len();
-						let _ = client.import_block(block);
+					let block = Unverified::from_rlp(b);
-						client.flush_queue();
+					match block {
+						Ok(block) => {
+							let num = block.header.number();
+							let hash = block.hash();
+							trace!(target: "json-tests", "{} – Importing {} bytes. Block #{}/{}", name, bytes_len, num, hash);
+							let res = client.import_block(block);
+							if let Err(e) = res {
+								warn!(target: "json-tests", "{} – Error importing block #{}/{}: {:?}", name, num, hash, e);
+							}
+							client.flush_queue();
+						},
+						Err(decoder_err) => {
+							warn!(target: "json-tests", "Error decoding test block: {:?} ({} bytes)", decoder_err, bytes_len);
+						}
 					}
 				}
 				fail_unless(client.chain_info().best_block_hash == blockchain.best_block.into());
@@ -108,24 +124,31 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 
 		if !fail {
 			flushln!("ok");
+		} else {
+			flushln!("fail");
 		}
 
 		start_stop_hook(&name, HookType::OnStop);
 	}
 
-	println!("!!! {:?} tests from failed.", failed.len());
+	if failed.len() > 0 {
+		println!("!!! {:?} tests failed.", failed.len());
+	}
 	failed
 }
 
 #[cfg(test)]
 mod block_tests {
+	use std::path::Path;
+
 	use super::json_chain_test;
 	use json_tests::HookType;
 
-	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], h: &mut H) -> Vec<String> {
-		json_chain_test(json_data, h)
+		json_chain_test(path, json_data, h)
 	}
-
+	//todo[dvdplm] do these tests match all folders in `res/` or are there tests we're missing?
+	//Issue: https://github.com/paritytech/parity-ethereum/issues/11085
 	declare_test!{BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"}
 	declare_test!{BlockchainTests_bcExploitTest, "BlockchainTests/bcExploitTest"}
 	declare_test!{BlockchainTests_bcForgedTest, "BlockchainTests/bcForgedTest"}
@@ -172,7 +195,12 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"}
-	declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
+	declare_test!{ BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
+	// todo[dvdplm]:
+	//      "RevertPrecompiledTouch_storage" contains 4 tests, only two fails
+	//      "RevertPrecompiledTouchExactOOG" contains a ton of tests, only two fails
+	//      "RevertPrecompiledTouch" has 4 tests, 2 failures
+	//  Ignored in currents.json, issue: https://github.com/paritytech/parity-ethereum/issues/11073
 	declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"}

ethcore/src/json_tests/difficulty.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
+
 use ethereum_types::U256;
 use ethjson::test_helpers::difficulty::DifficultyTest;
 use types::header::Header;
@@ -22,12 +24,14 @@ use spec::Spec;
 use super::HookType;
 
 pub fn json_difficulty_test<H: FnMut(&str, HookType)>(
+	path: &Path,
 	json_data: &[u8],
 	spec: Spec,
 	start_stop_hook: &mut H
 ) -> Vec<String> {
 	let _ = env_logger::try_init();
-	let tests = DifficultyTest::load(json_data).unwrap();
+	let tests = DifficultyTest::load(json_data)
+		.expect(&format!("Could not parse JSON difficulty test data from {}", path.display()));
 	let engine = &spec.engine;
 
 	for (name, test) in tests.into_iter() {
@@ -59,13 +63,14 @@ pub fn json_difficulty_test<H: FnMut(&str, HookType)>(
 macro_rules! difficulty_json_test {
 	( $spec:ident ) => {
 
+	use std::path::Path;
 	use super::json_difficulty_test;
 	use tempdir::TempDir;
 	use json_tests::HookType;
 
-	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], h: &mut H) -> Vec<String> {
 		let tempdir = TempDir::new("").unwrap();
-		json_difficulty_test(json_data, crate::spec::$spec(&tempdir.path()), h)
+		json_difficulty_test(path, json_data, crate::spec::$spec(&tempdir.path()), h)
 	}
 
 	}
@@ -73,12 +78,13 @@ macro_rules! difficulty_json_test {
 
 macro_rules! difficulty_json_test_nopath {
 	( $spec:ident ) => {
+	use std::path::Path;
 
 	use super::json_difficulty_test;
 	use json_tests::HookType;
 
-	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], h: &mut H) -> Vec<String> {
-		json_difficulty_test(json_data, crate::spec::$spec(), h)
+		json_difficulty_test(path, json_data, crate::spec::$spec(), h)
 	}
 
 	}

ethcore/src/json_tests/executive.rs

@@ -235,20 +235,22 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B>
 	}
 }
 
-fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], h: &mut H) -> Vec<String> {
 	let vms = VMType::all();
 	vms
 		.iter()
-		.flat_map(|vm| do_json_test_for(vm, json_data, h))
+		.flat_map(|vm| do_json_test_for(path, vm, json_data, h))
 		.collect()
 }
 
 fn do_json_test_for<H: FnMut(&str, HookType)>(
+	path: &Path,
 	vm_type: &VMType,
 	json_data: &[u8],
 	start_stop_hook: &mut H
 ) -> Vec<String> {
-	let tests = ethjson::test_helpers::vm::Test::load(json_data).unwrap();
+	let tests = ethjson::test_helpers::vm::Test::load(json_data)
+		.expect(&format!("Could not parse JSON executive test data from {}", path.display()));
 	let mut failed = Vec::new();
 
 	for (name, vm) in tests.into_iter() {

ethcore/src/json_tests/mod.rs

@@ -30,17 +30,7 @@ mod skip;
 mod difficulty;
 
 pub use self::test_common::HookType;
-
-pub use self::transaction::run_test_path as run_transaction_test_path;
-pub use self::transaction::run_test_file as run_transaction_test_file;
 pub use self::executive::run_test_path as run_executive_test_path;
 pub use self::executive::run_test_file as run_executive_test_file;
-pub use self::state::run_test_path as run_state_test_path;
+
-pub use self::state::run_test_file as run_state_test_file;
+use self::skip::SKIP_TESTS;
-pub use self::chain::run_test_path as run_chain_test_path;
-pub use self::chain::run_test_file as run_chain_test_file;
-pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
-pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
-pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
-pub use self::trie::run_secure_test_file as run_secure_trie_test_file;
-use self::skip::SKIP_TEST_STATE;

ethcore/src/json_tests/skip.rs

@@ -14,21 +14,30 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
-//! State tests to skip.
+//! State or blockchain tests to skip.
+//!
+//! Looks in the `ethereum/tests/test-issues/currents.json` file. This file contains two
+//! collections, `block` and `state`, each with a different format to specify single tests to skip.
+//!
+//! To skip a blockchain test, add a JSON object to the `block` array, where `failing` names the
+//! leaf folder with the tests to skip. The `subtests` array contains the names of the tests to skip.
+//! Note that this does not handle duplicate folder names, e.g. `ValidBlocks/funTests/` and
+//! `Something/funTests` would both be matched when `failing` is set to `funTests`.
+//!
+//! To skip a state test, add a JSON object to the `state` array. The `failing` works like for block
+//! tests, but the `subtests` key is an object on the form:
+//! "testName": {"subnumbers": [INDEX_OF_SKIPPED_SUBTESTS | "*"], "chain": "Blockchain name (informational)"}`
+//!
+//! Use the `reference` key to point to the github issue tracking to solution to the problem.
+//!
+//! Note: the `declare_test!` macro can also be use to skip tests, but skips entire files rather
+//! than single tests.
 
-use ethjson::test_helpers::skip::SkipStates;
+use ethjson::test_helpers::skip::SkipTests;
 
-#[cfg(feature="ci-skip-tests")]
 lazy_static! {
-	pub static ref SKIP_TEST_STATE: SkipStates = {
+	pub static ref SKIP_TESTS: SkipTests = {
 		let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json");
-		SkipStates::load(&skip_data[..]).expect("No invalid json allowed")
+		SkipTests::load(&skip_data[..]).expect("JSON from disk is valid")
-	};
-}
-
-#[cfg(not(feature="ci-skip-tests"))]
-lazy_static!{
-	pub static ref SKIP_TEST_STATE: SkipStates = {
-		SkipStates::empty()
 	};
 }

ethcore/src/json_tests/state.rs

@@ -22,34 +22,31 @@ use ethjson;
 use test_helpers::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess};
 use types::transaction::SignedTransaction;
 use vm::EnvInfo;
-use super::SKIP_TEST_STATE;
+use super::SKIP_TESTS;
 use super::HookType;
 
-/// Run state jsontests on a given folder.
+#[allow(dead_code)]
-pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
-	::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
-}
-
-/// Run state jsontests on a given file.
-pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
-	::json_tests::test_common::run_test_file(p, json_chain_test, h)
-}
-
 fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
-	SKIP_TEST_STATE.state.iter().any(|state_test|{
+	trace!(target: "json-tests", "[state, skip_test] subname: '{}', chain: '{}', number: {}", subname, chain, number);
+	SKIP_TESTS.state.iter().any(|state_test|{
 		if let Some(subtest) = state_test.subtests.get(subname) {
+			trace!(target: "json-tests", "[state, skip_test] Maybe skipping {:?}", subtest);
 			chain == &subtest.chain &&
-			(subtest.subnumbers[0] == "*"
+			(
-				 || subtest.subnumbers.contains(&number.to_string()))
+				subtest.subnumbers[0] == "*" ||
+				subtest.subnumbers.contains(&number.to_string())
+			)
 		} else {
 			false
 		}
 	})
 }
 
-pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
+#[allow(dead_code)]
+pub fn json_chain_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	let _ = ::env_logger::try_init();
-	let tests = ethjson::test_helpers::state::Test::load(json_data).unwrap();
+	let tests = ethjson::test_helpers::state::Test::load(json_data)
+		.expect(&format!("Could not parse JSON state test data from {}", path.display()));
 	let mut failed = Vec::new();
 
 	for (name, test) in tests.into_iter() {
@@ -65,7 +62,7 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 				let spec = match EvmTestClient::fork_spec_from_json(&spec_name) {
 					Some(spec) => spec,
 					None => {
-						println!("   - {} | {:?} Ignoring tests because of missing spec", name, spec_name);
+						println!("   - {} | {:?} Ignoring tests because of missing chainspec", name, spec_name);
 						continue;
 					}
 				};
@@ -73,7 +70,7 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 				for (i, state) in states.into_iter().enumerate() {
 					let info = format!("   - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total);
 					if skip_test(&name, &spec.name, i + 1) {
-						println!("{} in skip list : SKIPPED", info);
+						println!("{}: SKIPPED", info);
 						continue;
 					}
 
@@ -123,11 +120,13 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 
 #[cfg(test)]
 mod state_tests {
+	use std::path::Path;
+
 	use super::json_chain_test;
 	use json_tests::HookType;
 
-	fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], h: &mut H) -> Vec<String> {
-		json_chain_test(json_data, h)
+		json_chain_test(path, json_data, h)
 	}
 
 	declare_test!{GeneralStateTest_stArgsZeroOneBalance, "GeneralStateTests/stArgsZeroOneBalance/"}
@@ -164,6 +163,15 @@ mod state_tests {
 	declare_test!{GeneralStateTest_stRecursiveCreate, "GeneralStateTests/stRecursiveCreate/"}
 	declare_test!{GeneralStateTest_stRefundTest, "GeneralStateTests/stRefundTest/"}
 	declare_test!{GeneralStateTest_stReturnDataTest, "GeneralStateTests/stReturnDataTest/"}
+	// todo[dvdplm]:
+	//      "RevertPrecompiledTouch_storage" contains 4 tests, only two fails
+	//      "RevertPrecompiledTouchExactOOG" contains a ton of tests, only two fails
+	//      "RevertPrecompiledTouch" has 4 tests, 2 failures
+	//      Ignored in `currents.json`.
+	//      Issues:
+	//          https://github.com/paritytech/parity-ethereum/issues/11078
+	//          https://github.com/paritytech/parity-ethereum/issues/11079
+	//          https://github.com/paritytech/parity-ethereum/issues/11080
 	declare_test!{GeneralStateTest_stRevertTest, "GeneralStateTests/stRevertTest/"}
 	declare_test!{GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"}
 	declare_test!{GeneralStateTest_stShift, "GeneralStateTests/stShift/"}

ethcore/src/json_tests/test_common.rs

@@ -30,43 +30,61 @@ pub enum HookType {
 	OnStop
 }
 
+/// Run all tests under the given path (except for the test files named in the skip list) using the
+/// provided runner function.
 pub fn run_test_path<H: FnMut(&str, HookType)>(
-	p: &Path, skip: &[&'static str],
+	path: &Path,
-	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	skip: &[&'static str],
+	runner: fn(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H
 ) {
+	if !skip.is_empty() {
+		// todo[dvdplm] it's really annoying to have to use flushln here. Should be `info!(target:
+		// "json-tests", …)`. Issue https://github.com/paritytech/parity-ethereum/issues/11084
+		flushln!("[run_test_path] Skipping tests in {}: {:?}", path.display(), skip);
+	}
 	let mut errors = Vec::new();
-	run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors);
+	run_test_path_inner(path, skip, runner, start_stop_hook, &mut errors);
 	let empty: [String; 0] = [];
-	assert_eq!(errors, empty);
+	assert_eq!(errors, empty, "\nThere were {} tests in '{}' that failed.", errors.len(), path.display());
 }
 
 fn run_test_path_inner<H: FnMut(&str, HookType)>(
-	p: &Path, skip: &[&'static str],
+	p: &Path,
-	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	skip: &[&'static str],
+	runner: fn(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H,
 	errors: &mut Vec<String>
 ) {
 	let path = Path::new(p);
-	let s: HashSet<OsString> = skip.iter().map(|s| {
+	let extension = path.extension().and_then(|s| s.to_str());
+	let skip_list: HashSet<OsString> = skip.iter().map(|s| {
 		let mut os: OsString = s.into();
 		os.push(".json");
 		os
 	}).collect();
-	let extension = path.extension().and_then(|s| s.to_str());
+
 	if path.is_dir() {
-		for p in read_dir(path).unwrap().filter_map(|e| {
+		trace!(target: "json-tests", "running tests contained in '{}'", path.display());
-			let e = e.unwrap();
+		let test_files = read_dir(path)
-			if s.contains(&e.file_name()) {
+			.expect("Directory exists on disk")
-				None
+			.filter_map(|dir_entry| {
-			} else {
+				let dir_entry = dir_entry.expect("Entry in directory listing exists");
-				Some(e.path())
+				if skip_list.contains(&dir_entry.file_name()) {
-			}}) {
+					debug!(target: "json-tests", "'{:?}' is on the skip list.", dir_entry.file_name());
-			run_test_path_inner(&p, skip, runner, start_stop_hook, errors);
+					None
+				} else {
+					Some(dir_entry.path())
+				}
+			});
+		for test_file in test_files {
+			run_test_path_inner(&test_file, skip, runner, start_stop_hook, errors);
 		}
 	} else if extension == Some("swp") || extension == None {
+		trace!(target: "json-tests", "ignoring '{}', extension {:?} – Junk?", path.display(), extension);
 		// Ignore junk
 	} else {
+		trace!(target: "json-tests", "running tests in '{}'", path.display());
 		let mut path = p.to_path_buf();
 		path.set_extension("json");
 		run_test_file_append(&path, runner, start_stop_hook, errors)
@@ -75,7 +93,7 @@ fn run_test_path_inner<H: FnMut(&str, HookType)>(
 
 fn run_test_file_append<H: FnMut(&str, HookType)>(
 	path: &Path,
-	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	runner: fn(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H,
 	errors: &mut Vec<String>
 ) {
@@ -85,12 +103,12 @@ fn run_test_file_append<H: FnMut(&str, HookType)>(
 		Err(_) => panic!("Error opening test file at: {:?}", path),
 	};
 	file.read_to_end(&mut data).expect("Error reading test file");
-	errors.append(&mut runner(&data, start_stop_hook));
+	errors.append(&mut runner(&path, &data, start_stop_hook));
 }
 
 pub fn run_test_file<H: FnMut(&str, HookType)>(
 	path: &Path,
-	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	runner: fn(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H
 ) {
 	let mut data = Vec::new();
@@ -99,7 +117,7 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(
 		Err(_) => panic!("Error opening test file at: {:?}", path),
 	};
 	file.read_to_end(&mut data).expect("Error reading test file");
-	let results = runner(&data, start_stop_hook);
+	let results = runner(&path, &data, start_stop_hook);
 	let empty: [String; 0] = [];
 	assert_eq!(results, empty);
 }
@@ -107,11 +125,25 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(
 #[cfg(test)]
 macro_rules! test {
 	($name: expr, $skip: expr) => {
-		::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test, &mut |_, _| ());
+		::json_tests::test_common::run_test_path(
+			::std::path::Path::new(concat!("res/ethereum/tests/", $name)),
+			&$skip,
+			do_json_test,
+			&mut |_, _| ()
+		);
 	}
 }
 
-/// Declares a test
+/// Declares a test:
+///
+/// declare_test!(test_name, "path/to/folder/with/tests");
+///
+/// Declares a test but skip the named test files inside the folder (no extension):
+///
+/// declare_test!(skip => ["a-test-file", "other-test-file"], test_name, "path/to/folder/with/tests");
+///
+/// NOTE: a skipped test is considered a passing test as far as `cargo test` is concerned. Normally
+/// one test corresponds to a folder full of test files, each of which may contain many tests.
 #[macro_export]
 macro_rules! declare_test {
 	(skip => $arr: expr, $id: ident, $name: expr) => {

ethcore/src/json_tests/transaction.rs

@@ -26,22 +26,14 @@ use types::{
 };
 use machine::transaction_ext::Transaction;
 
-/// Run transaction jsontests on a given folder.
+#[allow(dead_code)]
-pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
-	::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+	// Block number used to run the tests.
-}
+	// Make sure that all the specified features are activated.
+	const BLOCK_NUMBER: u64 = 0x6ffffffffffffe;
 
-/// Run transaction jsontests on a given file.
+	let tests = ethjson::test_helpers::transaction::Test::load(json_data)
-pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
+		.expect(&format!("Could not parse JSON transaction test data from {}", path.display()));
-	::json_tests::test_common::run_test_file(p, do_json_test, h)
-}
-
-// Block number used to run the tests.
-// Make sure that all the specified features are activated.
-const BLOCK_NUMBER: u64 = 0x6ffffffffffffe;
-
-fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
-	let tests = ethjson::test_helpers::transaction::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 	for (name, test) in tests.into_iter() {
 		start_stop_hook(&name, HookType::OnStart);

ethcore/src/json_tests/trie.rs

@@ -14,19 +14,18 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::path::Path;
+
 use ethjson;
 use trie::{TrieFactory, TrieSpec};
 use ethereum_types::H256;
 
 use super::HookType;
 
-pub use self::generic::run_test_path as run_generic_test_path;
+#[allow(dead_code)]
-pub use self::generic::run_test_file as run_generic_test_file;
+fn test_trie<H: FnMut(&str, HookType)>(path: &Path, json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec<String> {
-pub use self::secure::run_test_path as run_secure_test_path;
+	let tests = ethjson::test_helpers::trie::Test::load(json)
-pub use self::secure::run_test_file as run_secure_test_file;
+		.expect(&format!("Could not parse JSON trie test data from {}", path.display()));
-
-fn test_trie<H: FnMut(&str, HookType)>(json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec<String> {
-	let tests = ethjson::test_helpers::trie::Test::load(json).unwrap();
 	let factory = TrieFactory::new(trie, ethtrie::Layout);
 	let mut result = vec![];
 
@@ -64,18 +63,9 @@ mod generic {
 
 	use super::HookType;
 
-	/// Run generic trie jsontests on a given folder.
+	#[allow(dead_code)]
-	pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json: &[u8], h: &mut H) -> Vec<String> {
-		::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+		super::test_trie(path, json, TrieSpec::Generic, h)
-	}
-
-	/// Run generic trie jsontests on a given file.
-	pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
-		::json_tests::test_common::run_test_file(p, do_json_test, h)
-	}
-
-	fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
-		super::test_trie(json, TrieSpec::Generic, h)
 	}
 
 	declare_test!{TrieTests_trietest, "TrieTests/trietest"}
@@ -88,18 +78,9 @@ mod secure {
 
 	use super::HookType;
 
-	/// Run secure trie jsontests on a given folder.
+	#[allow(dead_code)]
-	pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
+	fn do_json_test<H: FnMut(&str, HookType)>(path: &Path, json: &[u8], h: &mut H) -> Vec<String> {
-		::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
+		super::test_trie(path, json, TrieSpec::Secure, h)
-	}
-
-	/// Run secure trie jsontests on a given file.
-	pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
-		::json_tests::test_common::run_test_file(p, do_json_test, h)
-	}
-
-	fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
-		super::test_trie(json, TrieSpec::Secure, h)
 	}
 
 	declare_test!{TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"}

json/src/maybe.rs

@@ -18,9 +18,13 @@
 
 use std::fmt;
 use std::marker::PhantomData;
+
+use ethereum_types::U256;
 use serde::{Deserialize, Deserializer};
 use serde::de::{Error, Visitor, IntoDeserializer};
 
+use crate::uint::Uint;
+
 /// Deserializer of empty string values into optionals.
 #[derive(Debug, PartialEq, Clone)]
 pub enum MaybeEmpty<T> {
@@ -32,7 +36,8 @@ pub enum MaybeEmpty<T> {
 
 impl<'a, T> Deserialize<'a> for MaybeEmpty<T> where T: Deserialize<'a> {
 	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-		where D: Deserializer<'a> {
+		where D: Deserializer<'a>
+	{
 		deserializer.deserialize_any(MaybeEmptyVisitor::new())
 	}
 }
@@ -78,6 +83,37 @@ impl<T> Into<Option<T>> for MaybeEmpty<T> {
 	}
 }
 
+#[cfg(test)]
+impl From<Uint> for MaybeEmpty<Uint> {
+	fn from(uint: Uint) -> Self {
+		MaybeEmpty::Some(uint)
+	}
+}
+
+impl From<MaybeEmpty<Uint>> for U256 {
+	fn from(maybe: MaybeEmpty<Uint>) -> U256 {
+		match maybe {
+			MaybeEmpty::Some(v) => v.0,
+			MaybeEmpty::None => U256::zero(),
+		}
+	}
+}
+
+impl From<MaybeEmpty<Uint>> for u64 {
+	fn from(maybe: MaybeEmpty<Uint>) -> u64 {
+		match maybe {
+			MaybeEmpty::Some(v) => v.0.low_u64(),
+			MaybeEmpty::None => 0u64,
+		}
+	}
+}
+
+impl Default for MaybeEmpty<Uint> {
+	fn default() -> Self {
+		MaybeEmpty::Some(Uint::default())
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use std::str::FromStr;

json/src/spec/mod.rs

@@ -40,7 +40,7 @@ pub use self::params::Params;
 pub use self::spec::{Spec, ForkSpec};
 pub use self::seal::{Seal, Ethereum, AuthorityRoundSeal, TendermintSeal};
 pub use self::engine::Engine;
-pub use self::state::State;
+pub use self::state::{State, HashOrMap};
 pub use self::ethash::{Ethash, EthashParams, BlockReward};
 pub use self::validator_set::ValidatorSet;
 pub use self::basic_authority::{BasicAuthority, BasicAuthorityParams};

json/src/spec/state.rs

@@ -19,38 +19,55 @@
 use std::collections::BTreeMap;
 use crate::{
 	bytes::Bytes,
-	hash::Address,
+	hash::{Address, H256},
 	spec::{Account, Builtin}
 };
 use serde::Deserialize;
 
-/// Blockchain state deserializer for tests
+/// Recent JSON tests can be either a map or a hash (represented by a string).
-#[cfg(any(test, feature = "test-helpers"))]
+/// See https://github.com/ethereum/tests/issues/637
-#[derive(Clone, Debug, PartialEq, Deserialize)]
+#[cfg_attr(any(test, feature = "test-helpers"), derive(Clone))]
-#[serde(deny_unknown_fields)]
+#[derive(Debug, PartialEq, Deserialize)]
-pub struct State(pub BTreeMap<Address, Account>);
+#[serde(untagged)]
+pub enum HashOrMap {
+	/// When the `postState` is large, tests sometimes just include the state root of the last
+	/// successful block here.
+	Hash(H256),
+	/// The expected `postState` of a test
+	Map(BTreeMap<Address, Account>),
+}
 
 /// Blockchain state deserializer.
-#[cfg(not(any(test, feature = "test-helpers")))]
+#[cfg_attr(any(test, feature = "test-helpers"), derive(Clone))]
 #[derive(Debug, PartialEq, Deserialize)]
 #[serde(deny_unknown_fields)]
-pub struct State(BTreeMap<Address, Account>);
+pub struct State(pub HashOrMap);
 
 impl State {
 	/// Returns all builtins.
 	pub fn builtins(&self) -> BTreeMap<Address, Builtin> {
-		self.0
+		match &self.0 {
-			.iter()
+			HashOrMap::Hash(_) => BTreeMap::default(),
-			.filter_map(|(add, ref acc)| acc.builtin.clone().map(|b| (add.clone(), b)))
+			HashOrMap::Map(map) => {
-			.collect()
+				map.iter().filter_map(|(add, ref acc)| {
+					acc.builtin.clone().map(|b| (add.clone(), b))
+				}).collect()
+			}
+
+		}
 	}
 
 	/// Returns all constructors.
 	pub fn constructors(&self) -> BTreeMap<Address, Bytes> {
-		self.0
+		match &self.0 {
-			.iter()
+			HashOrMap::Hash(_) => BTreeMap::default(),
-			.filter_map(|(add, ref acc)| acc.constructor.clone().map(|b| (add.clone(), b)))
+			HashOrMap::Map(map) => {
-			.collect()
+				map.iter().filter_map(|(add, ref acc)| {
+					acc.constructor.clone().map(|b| (add.clone(), b))
+				}).collect()
+			}
+
+		}
 	}
 }
 
@@ -59,6 +76,10 @@ impl IntoIterator for State {
 	type IntoIter = <BTreeMap<Address, Account> as IntoIterator>::IntoIter;
 
 	fn into_iter(self) -> Self::IntoIter {
-		self.0.into_iter()
+		if let HashOrMap::Map(m) = self.0 {
+			m.into_iter()
+		} else {
+			BTreeMap::default().into_iter()
+		}
 	}
 }

json/src/test_helpers/skip.rs

@@ -3,17 +3,17 @@ use serde::Deserialize;
 
 /// Test to skip (only if issue ongoing)
 #[derive(Debug, PartialEq, Deserialize)]
-pub struct SkipStates {
+pub struct SkipTests {
 	/// Block tests
-	pub block: Vec<BlockSkipStates>,
+	pub block: Vec<SkipBlockchainTest>,
 	/// State tests
-	pub state: Vec<StateSkipStates>,
+	pub state: Vec<SkipStateTest>,
 
 }
 
 /// Block test to skip.
 #[derive(Debug, PartialEq, Deserialize)]
-pub struct BlockSkipStates {
+pub struct SkipBlockchainTest {
 	/// Issue reference.
 	pub reference: String,
 	/// Test failing name.
@@ -24,7 +24,7 @@ pub struct BlockSkipStates {
 
 /// State test to skip.
 #[derive(Debug, PartialEq, Deserialize)]
-pub struct StateSkipStates {
+pub struct SkipStateTest {
 	/// Issue reference.
 	pub reference: String,
 	/// Test failing name.
@@ -42,10 +42,10 @@ pub struct StateSkipSubStates {
 	pub chain: String,
 }
 
-impl SkipStates {
+impl SkipTests {
 	/// Empty skip states.
 	pub fn empty() -> Self {
-		SkipStates {
+		SkipTests {
 			block: Vec::new(),
 			state: Vec::new(),
 		}

json/src/test_helpers/transaction.rs

@@ -26,8 +26,11 @@ pub type Test = super::tester::GenericTester<String, TransactionTest>;
 /// Transaction test deserialization.
 #[derive(Debug, Deserialize)]
 pub struct TransactionTest {
+	/// RLP of the transaction
 	pub rlp: Bytes,
+	#[allow(missing_docs)]
 	pub _info: serde::de::IgnoredAny,
+	/// State of the transaction after the test runs
 	#[serde(flatten)]
 	pub post_state: BTreeMap<ForkSpec, PostState>,
 }

json/src/transaction.rs

@@ -36,11 +36,14 @@ pub struct Transaction {
 	/// Value.
 	pub value: Uint,
 	/// R.
-	pub r: Uint,
+	#[serde(default)]
+	pub r: MaybeEmpty<Uint>,
 	/// S.
-	pub s: Uint,
+	#[serde(default)]
+	pub s: MaybeEmpty<Uint>,
 	/// V.
-	pub v: Uint,
+	#[serde(default)]
+	pub v: MaybeEmpty<Uint>,
 	/// Secret
 	#[serde(rename = "secretKey")]
 	pub secret: Option<H256>,
@@ -60,21 +63,21 @@ mod tests {
 			"nonce" : "0x00",
 			"to" : "",
 			"value" : "0x00",
-			"r": 0,
+			"r": "0",
-			"s": 1,
+			"s": "1",
-			"v": 2,
+			"v": "2",
 			"secretKey": "0x0000000000000000000000000000000000000000000000000000000000000000"
 		}"#;
-		let tx: Transaction = serde_json::from_str(s).unwrap();
+		let tx: Transaction = serde_json::from_str(s).expect("JSON string is valid");
 		assert_eq!(tx.data, Bytes::new(Vec::new()));
 		assert_eq!(tx.gas_limit, Uint(U256::from(0xf388)));
 		assert_eq!(tx.gas_price, Uint(U256::from(0x09184e72a000_u64)));
 		assert_eq!(tx.nonce, Uint(U256::zero()));
 		assert_eq!(tx.to, MaybeEmpty::None);
 		assert_eq!(tx.value, Uint(U256::zero()));
-		assert_eq!(tx.r, Uint(U256::zero()));
+		assert_eq!(tx.r, Uint(U256::zero()).into());
-		assert_eq!(tx.s, Uint(U256::one()));
+		assert_eq!(tx.s, Uint(U256::one()).into());
-		assert_eq!(tx.v, Uint(U256::from(2)));
+		assert_eq!(tx.v, Uint(U256::from(2)).into());
 		assert_eq!(tx.secret, Some(H256(Eth256::zero())));
 	}
 }

json/src/vm.rs

@@ -127,7 +127,7 @@ mod tests {
 	};
 	use super::{Address, Bytes, Call, Env, H256, MaybeEmpty, State, Transaction, Uint, Vm};
 
-	use crate::spec::Account;
+	use crate::spec::{Account, HashOrMap};
 	use ethereum_types::{U256, H160 as Hash160, H256 as Hash256};
 	use macros::map;
 	use rustc_hex::FromHex;
@@ -179,7 +179,7 @@ mod tests {
 				}
 			}
 		}"#;
-		let vm: Vm = serde_json::from_str(s).unwrap();
+		let vm: Vm = serde_json::from_str(s).expect("JSON is valid");
 		assert_eq!(vm.calls, Some(Vec::new()));
 		assert_eq!(vm.env, Env {
 			author: Address(Hash160::from_str("2adc25665018aa1fe0e6bc666dac8fc2697ff9ba").unwrap()),
@@ -205,31 +205,36 @@ mod tests {
 			Some(H256(Hash256::from_str("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347").unwrap()))
 		);
 		assert_eq!(vm.output, Some(Bytes::new(Vec::new())));
-		assert_eq!(vm.pre_state, State(map![
+		assert_eq!(vm.pre_state, State(
-			Address(Hash160::from_str("0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6").unwrap()) => Account {
+			HashOrMap::Map(
-				builtin: None,
+				map![
-				balance: Some(Uint(0x0de0b6b3a7640000_u64.into())),
-				code: Some(Bytes::new(TEST_CODE.from_hex().unwrap())),
-				constructor: None,
-				nonce: Some(Uint(0.into())),
-				storage: Some(map![]),
-				version: None,
-			}])
-		);
-		assert_eq!(vm.post_state, Some(
-				State(map![
 					Address(Hash160::from_str("0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6").unwrap()) => Account {
 						builtin: None,
 						balance: Some(Uint(0x0de0b6b3a7640000_u64.into())),
 						code: Some(Bytes::new(TEST_CODE.from_hex().unwrap())),
 						constructor: None,
 						nonce: Some(Uint(0.into())),
-						storage: Some(map![
+						storage: Some(map![]),
-							Uint(0.into()) => Uint(U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe").unwrap())
-						]),
 						version: None,
-				}])
+					}
-			)
+				]))
+		);
+		assert_eq!(vm.post_state, Some(
+				State(
+					HashOrMap::Map(
+						map![
+							Address(Hash160::from_str("0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6").unwrap()) => Account {
+								builtin: None,
+								balance: Some(Uint(0x0de0b6b3a7640000_u64.into())),
+								code: Some(Bytes::new(TEST_CODE.from_hex().unwrap())),
+								constructor: None,
+								nonce: Some(Uint(0.into())),
+								storage: Some(map![
+									Uint(0.into()) => Uint(U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe").unwrap())
+								]),
+								version: None,
+						}]))
+				)
 		);
 	}
 

rpc/src/v1/tests/eth.rs

@@ -174,13 +174,13 @@ impl EthTester {
 
 #[test]
 fn harness_works() {
-	let chain: BlockChain = extract_chain!("BlockchainTests/bcWalletTest/wallet2outOf3txs");
+	let chain: BlockChain = extract_chain!("BlockchainTests/ValidBlocks/bcWalletTest/wallet2outOf3txs");
 	let _ = EthTester::from_chain(&chain);
 }
 
 #[test]
 fn eth_get_balance() {
-	let chain = extract_chain!("BlockchainTests/bcWalletTest/wallet2outOf3txs");
+	let chain = extract_chain!("BlockchainTests/ValidBlocks/bcWalletTest/wallet2outOf3txs");
 	let tester = EthTester::from_chain(&chain);
 	// final account state
 	let req_latest = r#"{
@@ -206,7 +206,7 @@ fn eth_get_balance() {
 
 #[test]
 fn eth_get_proof() {
-	let chain = extract_chain!("BlockchainTests/bcWalletTest/wallet2outOf3txs");
+	let chain = extract_chain!("BlockchainTests/ValidBlocks/bcWalletTest/wallet2outOf3txs");
 	let tester = EthTester::from_chain(&chain);
 	// final account state
 	let req_latest = r#"{
@@ -232,7 +232,7 @@ fn eth_get_proof() {
 
 #[test]
 fn eth_block_number() {
-	let chain = extract_chain!("BlockchainTests/bcGasPricerTest/RPC_API_Test");
+	let chain = extract_chain!("BlockchainTests/ValidBlocks/bcGasPricerTest/RPC_API_Test");
 	let tester = EthTester::from_chain(&chain);
 	let req_number = r#"{
 		"jsonrpc": "2.0",
@@ -247,7 +247,7 @@ fn eth_block_number() {
 
 #[test]
 fn eth_get_block() {
-	let chain = extract_chain!("BlockchainTests/bcGasPricerTest/RPC_API_Test");
+	let chain = extract_chain!("BlockchainTests/ValidBlocks/bcGasPricerTest/RPC_API_Test");
 	let tester = EthTester::from_chain(&chain);
 	let req_block = r#"{"method":"eth_getBlockByNumber","params":["0x0",false],"id":1,"jsonrpc":"2.0"}"#;
 
@@ -257,13 +257,13 @@ fn eth_get_block() {
 
 #[test]
 fn eth_get_block_by_hash() {
-	let chain = extract_chain!("BlockchainTests/bcGasPricerTest/RPC_API_Test");
+	let chain = extract_chain!("BlockchainTests/ValidBlocks/bcGasPricerTest/RPC_API_Test");
 	let tester = EthTester::from_chain(&chain);
 
 	// We're looking for block number 4 from "RPC_API_Test_Frontier"
-	let req_block = r#"{"method":"eth_getBlockByHash","params":["0xaddb9e39795e9e041c936b88a2577802569f34afded0948707b074caa3163a87",false],"id":1,"jsonrpc":"2.0"}"#;
+	let req_block = r#"{"method":"eth_getBlockByHash","params":["0x75e65fb3bbf5f53afe26dcc72df6a95b0e8ca5f1c450145d8c3915bd0308b75b",false],"id":1,"jsonrpc":"2.0"}"#;
 
-	let res_block = r#"{"jsonrpc":"2.0","result":{"author":"0x8888f1f195afa192cfee860698584c030f4c9db1","difficulty":"0x20080","extraData":"0x","gasLimit":"0x1dd7ea0","gasUsed":"0x5458","hash":"0xaddb9e39795e9e041c936b88a2577802569f34afded0948707b074caa3163a87","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x8888f1f195afa192cfee860698584c030f4c9db1","mixHash":"0x713b0b31f6e72d8cb7367eaf59447ea531f209fc80e6379edd9f8d3bb73931c4","nonce":"0x4534b406bc23b86d","number":"0x4","parentHash":"0x17567aa5995b703736e32972289d68af50543acc4d56d37e8ad1fea7252cac4a","receiptsRoot":"0x7ed8026cf72ed0e98e6fd53ab406e51ffd34397d9da0052494ff41376fda7b5f","sealFields":["0xa0713b0b31f6e72d8cb7367eaf59447ea531f209fc80e6379edd9f8d3bb73931c4","0x884534b406bc23b86d"],"sha3Uncles":"0xe588a44b3e320e72e70b32b531f3ac0d432e756120135ae8fe5fa10895196b40","size":"0x661","stateRoot":"0x68805721294e365020aca15ed56c360d9dc2cf03cbeff84c9b84b8aed023bfb5","timestamp":"0x5bbdf772","totalDifficulty":"0xa00c0","transactions":["0xb094b9dc356dbb8b256402c6d5709288066ad6a372c90c9c516f14277545fd58"],"transactionsRoot":"0x97a593d8d7e15b57f5c6bb25bc6c325463ef99f874bc08a78656c3ab5cb23262","uncles":["0x86b48f5186c4b0882d3dca7977aa37840008832ef092f8ef797019dc74bfa8c7","0x2da9d062c11d536f0f1cc2a4e0111597c79926958d0fc26ae1a2d07d1a3bf47d"]},"id":1}"#;
+	let res_block = r#"{"jsonrpc":"2.0","result":{"author":"0x8888f1f195afa192cfee860698584c030f4c9db1","difficulty":"0x20000","extraData":"0x","gasLimit":"0x1dd7ea0","gasUsed":"0x5458","hash":"0x75e65fb3bbf5f53afe26dcc72df6a95b0e8ca5f1c450145d8c3915bd0308b75b","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x8888f1f195afa192cfee860698584c030f4c9db1","mixHash":"0x55553aaef7ee28e3aea539eb784e8cc26646911a19126c242ac682c3fcf22041","nonce":"0xca2904e50ca47ace","number":"0x4","parentHash":"0x58849f66c0ca60054468725cf173b72a2769807152c625aa02e71d67ab2eaed5","receiptsRoot":"0x7ed8026cf72ed0e98e6fd53ab406e51ffd34397d9da0052494ff41376fda7b5f","sealFields":["0xa055553aaef7ee28e3aea539eb784e8cc26646911a19126c242ac682c3fcf22041","0x88ca2904e50ca47ace"],"sha3Uncles":"0x0dbc9711185574f2eee337af18d08c0afe85490304c6bb16b443991b552c5e2c","size":"0x661","stateRoot":"0x68805721294e365020aca15ed56c360d9dc2cf03cbeff84c9b84b8aed023bfb5","timestamp":"0x5c477134","totalDifficulty":"0xa0000","transactions":["0xb094b9dc356dbb8b256402c6d5709288066ad6a372c90c9c516f14277545fd58"],"transactionsRoot":"0x97a593d8d7e15b57f5c6bb25bc6c325463ef99f874bc08a78656c3ab5cb23262","uncles":["0x51b0d7366382926a4f83191af19cb4aa894f6fd9bd1bda6c04de3d5af70eddba","0x9263e0be8311eb79db96171fad3fdd70317bbbdc4081ad6b04c60335db65a3bb"]},"id":1}"#;
 	assert_eq!(tester.handler.handle_request_sync(req_block).unwrap(), res_block);
 }
 
@@ -509,6 +509,6 @@ fn starting_nonce_test() {
 	assert_eq!(r#"{"jsonrpc":"2.0","result":"0x100","id":15}"#, &sample);
 }
 
-register_test!(eth_transaction_count_1, verify_transaction_counts, "BlockchainTests/bcWalletTest/wallet2outOf3txs");
+register_test!(eth_transaction_count_1, verify_transaction_counts, "BlockchainTests/ValidBlocks/bcWalletTest/wallet2outOf3txs");
-register_test!(eth_transaction_count_2, verify_transaction_counts, "BlockchainTests/bcTotalDifficultyTest/sideChainWithMoreTransactions");
+register_test!(eth_transaction_count_2, verify_transaction_counts, "BlockchainTests/ValidBlocks/bcTotalDifficultyTest/sideChainWithMoreTransactions");
-register_test!(eth_transaction_count_3, verify_transaction_counts, "BlockchainTests/bcGasPricerTest/RPC_API_Test");
+register_test!(eth_transaction_count_3, verify_transaction_counts, "BlockchainTests/ValidBlocks/bcGasPricerTest/RPC_API_Test");

scripts/gitlab/test-linux.sh

@@ -5,7 +5,7 @@ echo "________Running test-linux.sh________"
 set -e # fail on any error
 set -u # treat unset variables as error
 
-FEATURES="json-tests,ci-skip-tests"
+FEATURES="json-tests"
 OPTIONS="--release"
 #use nproc `linux only
 THREADS=$(nproc)