Add constantinople conf to EvmTestClient. (#9570)

* Add constantinople conf to EvmTestClient.

* Skip some test to update submodule etheureum/tests submodule to latest.

* Put skipping 'under issue' test behind a feature.

* Change blockReward for const-test to pass ethereum/tests

* Update tests to new constantinple definition (change of reward at block
5).
Switch 'reference' to string, that way we can include issues from others
repo (more flexible)Update tests to new constantinple definition (change
of reward at block 5).
Switch 'reference' to string, that way we can include issues from others
repo (more flexible).

* Fix modexp and bn128_mul gas prices in chain config

* Changes `run_test_path` method to append its directory results (without
that it stop testing at the first file failure).
Add some missing tests.
Add skip for those (block create2 is one hundred percent false but on
hive we can see that geth and aleth got similar issue for this item).

* retab current.json

* Update reference to parity issue for failing tests.

Cargo.toml

@@ -89,6 +89,7 @@ daemonize = { git = "https://github.com/paritytech/daemonize" }
 [features]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
+ci-skip-issue = ["ethcore/ci-skip-issue"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethcore/Cargo.toml

@@ -102,6 +102,8 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 slow-blocks = []
 # Run JSON consensus tests.
 json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"]
+# Skip JSON consensus tests with pending issues.
+ci-skip-issue = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile benches

ethcore/res/ethereum/constantinople_test.json

@@ -6,7 +6,10 @@
 				"minimumDifficulty": "0x020000",
 				"difficultyBoundDivisor": "0x0800",
 				"durationLimit": "0x0d",
-				"blockReward": "0x1BC16D674EC80000",
+				"blockReward": {
+					"0": "0x29A2241AF62C0000",
+					"5": "0x1BC16D674EC80000"
+				},
 				"homesteadTransition": "0x0",
 				"eip100bTransition": "0x0",
 				"difficultyBombDelays": {
@@ -58,9 +61,9 @@
 		"0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
 		"0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
 		"0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
-		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 100 } } } },
+		"0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", "activate_at": "0x00", "pricing": { "modexp": { "divisor": 20 } } } },
 		"0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", "activate_at": "0x00", "pricing": { "linear": { "base": 500, "word": 0 } } } },
-		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 2000, "word": 0 } } } },
+		"0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", "activate_at": "0x00", "pricing": { "linear": { "base": 40000, "word": 0 } } } },
 		"0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", "activate_at": "0x00", "pricing": { "alt_bn128_pairing": { "base": 100000, "pair": 80000 } } } }
 	}
 }

ethcore/res/ethereum/tests

@@ -1 +1 @@
-Subproject commit b8a21c193696976ca3b33b6d82107601063a5d26
+Subproject commit 3f5febc901913ef698f1b09dda8705babd729e4a

ethcore/res/ethereum/tests-issues/currents.json

@@ -0,0 +1,370 @@
+{ "block":
+	[
+		{
+			"reference": "9590",
+			"failing": "stCreateTest",
+			"subtests": [
+				"CreateOOGafterInitCodeReturndata2_d0g1v0_Constantinople"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "stCreate2",
+			"subtests": [
+				"RevertDepthCreateAddressCollision_d0g1v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d1g1v1_Constantinople",
+				"CREATE2_Suicide_d5g0v0_Constantinople",
+				"CREATE2_Suicide_d7g0v0_Constantinople",
+				"create2collisionSelfdestructedOOG_d2g0v0_Byzantium",
+				"create2collisionSelfdestructedOOG_d2g0v0_Constantinople",
+				"create2collisionNonce_d1g0v0_Byzantium",
+				"create2collisionNonce_d1g0v0_Constantinople",
+				"CreateMessageRevertedOOGInInit_d0g1v0_Constantinople",
+				"create2callPrecompiles_d3g0v0_Constantinople",
+				"create2collisionCode_d1g0v0_Byzantium",
+				"create2collisionCode_d1g0v0_Constantinople",
+				"create2collisionStorage_d0g0v0_Byzantium",
+				"create2collisionStorage_d0g0v0_Constantinople",
+				"create2callPrecompiles_d4g0v0_Constantinople",
+				"create2collisionSelfdestructedRevert_d0g0v0_Byzantium",
+				"create2collisionSelfdestructedRevert_d0g0v0_Constantinople",
+				"CreateMessageReverted_d0g1v0_Constantinople",
+				"RevertOpcodeCreate_d0g1v0_Constantinople",
+				"CREATE2_Suicide_d11g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d5g0v0_Constantinople",
+				"create2collisionSelfdestructedOOG_d1g0v0_Byzantium",
+				"create2collisionSelfdestructedOOG_d1g0v0_Constantinople",
+				"returndatacopy_following_create_d1g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d1g1v1_Constantinople",
+				"create2collisionSelfdestructed_d2g0v0_Byzantium",
+				"create2collisionSelfdestructed_d2g0v0_Constantinople",
+				"create2callPrecompiles_d2g0v0_Constantinople",
+				"create2InitCodes_d2g0v0_Constantinople",
+				"create2collisionNonce_d2g0v0_Byzantium",
+				"create2collisionNonce_d2g0v0_Constantinople",
+				"create2collisionCode_d0g0v0_Byzantium",
+				"create2collisionCode_d0g0v0_Constantinople",
+				"CREATE2_Bounds_d0g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d0g0v0_Constantinople",
+				"CREATE2_Suicide_d1g0v0_Constantinople",
+				"CREATE2_Bounds3_d0g1v0_Constantinople",
+				"create2collisionStorage_d2g0v0_Byzantium",
+				"create2collisionStorage_d2g0v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d0g0v1_Constantinople",
+				"create2callPrecompiles_d5g0v0_Constantinople",
+				"create2collisionCode2_d0g0v0_Byzantium",
+				"create2collisionCode2_d0g0v0_Constantinople",
+				"create2noCash_d0g0v0_Byzantium",
+				"create2noCash_d0g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d7g0v0_Constantinople",
+				"create2SmartInitCode_d1g0v0_Constantinople",
+				"create2InitCodes_d6g0v0_Constantinople",
+				"create2noCash_d1g0v0_Byzantium",
+				"create2noCash_d1g0v0_Constantinople",
+				"CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn_d0g0v0_Constantinople",
+				"RevertOpcodeInCreateReturns_d0g0v0_Constantinople",
+				"create2collisionStorage_d1g0v0_Byzantium",
+				"create2collisionStorage_d1g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d3g0v0_Constantinople",
+				"create2collisionBalance_d0g0v0_Byzantium",
+				"create2collisionBalance_d0g0v0_Constantinople",
+				"create2collisionSelfdestructed2_d0g0v0_Constantinople",
+				"create2InitCodes_d3g0v0_Constantinople",
+				"create2collisionCode2_d1g0v0_Byzantium",
+				"create2collisionCode2_d1g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d1g0v0_Constantinople",
+				"create2collisionBalance_d1g0v0_Byzantium",
+				"create2collisionBalance_d1g0v0_Constantinople",
+				"CREATE2_Bounds3_d0g2v0_Constantinople",
+				"create2callPrecompiles_d6g0v0_Constantinople",
+				"Create2Recursive_d0g0v0_Constantinople",
+				"create2collisionSelfdestructedOOG_d0g0v0_Byzantium",
+				"create2collisionSelfdestructedOOG_d0g0v0_Constantinople",
+				"CREATE2_Suicide_d3g0v0_Constantinople",
+				"returndatacopy_following_create_d0g0v0_Constantinople",
+				"create2InitCodes_d8g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d0g0v1_Constantinople",
+				"create2checkFieldsInInitcode_d2g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d1g0v1_Constantinople",
+				"Create2OnDepth1024_d0g0v0_Constantinople",
+				"create2collisionSelfdestructed2_d1g0v0_Constantinople",
+				"create2collisionSelfdestructedRevert_d2g0v0_Byzantium",
+				"create2collisionSelfdestructedRevert_d2g0v0_Constantinople",
+				"create2callPrecompiles_d0g0v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d0g1v1_Constantinople",
+				"create2collisionSelfdestructed_d1g0v0_Byzantium",
+				"create2collisionSelfdestructed_d1g0v0_Constantinople",
+				"call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Byzantium",
+				"call_outsize_then_create2_successful_then_returndatasize_d0g0v0_Constantinople",
+				"Create2OOGafterInitCodeRevert_d0g0v0_Constantinople",
+				"Create2OOGafterInitCodeReturndata3_d0g0v0_Constantinople",
+				"Create2OOGafterInitCodeReturndataSize_d0g0v0_Constantinople",
+				"create2InitCodes_d7g0v0_Constantinople",
+				"CREATE2_Suicide_d10g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d0g1v0_Constantinople",
+				"create2InitCodes_d5g0v0_Constantinople",
+				"create2collisionSelfdestructedRevert_d1g0v0_Byzantium",
+				"create2collisionSelfdestructedRevert_d1g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d1g1v0_Constantinople",
+				"create2collisionSelfdestructed_d0g0v0_Byzantium",
+				"create2collisionSelfdestructed_d0g0v0_Constantinople",
+				"create2noCash_d2g0v0_Byzantium",
+				"create2noCash_d2g0v0_Constantinople",
+				"CREATE2_Bounds3_d0g0v0_Constantinople",
+				"create2collisionNonce_d0g0v0_Byzantium",
+				"create2collisionNonce_d0g0v0_Constantinople",
+				"CREATE2_Suicide_d2g0v0_Constantinople",
+				"Create2OOGafterInitCode_d0g0v0_Constantinople",
+				"call_then_create2_successful_then_returndatasize_d0g0v0_Byzantium",
+				"call_then_create2_successful_then_returndatasize_d0g0v0_Constantinople",
+				"create2collisionBalance_d2g0v0_Byzantium",
+				"create2collisionBalance_d2g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d6g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d0g1v1_Constantinople",
+				"returndatacopy_afterFailing_create_d0g0v0_Constantinople",
+				"returndatacopy_following_revert_in_create_d0g0v0_Constantinople",
+				"CREATE2_Suicide_d9g0v0_Constantinople",
+				"create2callPrecompiles_d7g0v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d1g0v1_Constantinople",
+				"create2InitCodes_d1g0v0_Constantinople",
+				"CREATE2_Bounds_d0g1v0_Constantinople",
+				"Create2OOGafterInitCodeReturndata_d0g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d4g0v0_Constantinople",
+				"CreateMessageRevertedOOGInInit_d0g0v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d1g1v0_Constantinople",
+				"returndatacopy_following_successful_create_d0g0v0_Constantinople",
+				"create2checkFieldsInInitcode_d0g0v0_Constantinople",
+				"CreateMessageReverted_d0g0v0_Constantinople",
+				"create2SmartInitCode_d0g0v0_Constantinople",
+				"CREATE2_Bounds2_d0g0v0_Constantinople",
+				"returndatasize_following_successful_create_d0g0v0_Constantinople",
+				"CREATE2_Bounds2_d0g1v0_Constantinople",
+				"returndatacopy_0_0_following_successful_create_d0g0v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d0g0v0_Constantinople",
+				"CREATE2_Suicide_d0g0v0_Constantinople",
+				"create2InitCodes_d0g0v0_Constantinople",
+				"Create2OnDepth1023_d0g0v0_Constantinople",
+				"create2InitCodes_d4g0v0_Constantinople",
+				"Create2OOGafterInitCodeReturndata2_d0g0v0_Constantinople",
+				"create2collisionBalance_d3g0v0_Byzantium",
+				"create2collisionBalance_d3g0v0_Constantinople",
+				"CREATE2_Suicide_d4g0v0_Constantinople",
+				"Create2OOGafterInitCode_d0g1v0_Constantinople",
+				"RevertDepthCreateAddressCollision_d1g0v0_Constantinople",
+				"Create2OOGafterInitCodeRevert2_d0g0v0_Constantinople",
+				"Create2OOGafterInitCodeReturndata_d0g1v0_Constantinople",
+				"Create2Recursive_d0g1v0_Constantinople",
+				"create2collisionCode_d2g0v0_Byzantium",
+				"create2collisionCode_d2g0v0_Constantinople",
+				"CREATE2_Suicide_d6g0v0_Constantinople",
+				"CREATE2_Suicide_d8g0v0_Constantinople",
+				"RevertOpcodeCreate_d0g0v0_Constantinople",
+				"Create2OOGafterInitCodeReturndata2_d0g1v0_Constantinople",
+				"create2callPrecompiles_d1g0v0_Constantinople",
+				"RevertInCreateInInit_d0g0v0_Constantinople",
+				"RevertDepthCreate2OOG_d1g0v0_Constantinople"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "bcStateTest",
+			"subtests": [
+				"suicideStorageCheck_Byzantium",
+				"suicideStorageCheck_Constantinople",
+				"suicideStorageCheckVCreate2_Byzantium",
+				"suicideStorageCheckVCreate2_Constantinople",
+				"create2collisionwithSelfdestructSameBlock_Constantinople",
+				"blockhashNonConstArg_Constantinople",
+				"suicideThenCheckBalance_Constantinople",
+				"suicideThenCheckBalance_Homestead",
+				"suicideStorageCheckVCreate_Byzantium",
+				"suicideStorageCheckVCreate_Constantinople"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "stEIP158Specific",
+			"subtests": [
+				"callToEmptyThenCallError_d0g0v0_Byzantium",
+				"callToEmptyThenCallError_d0g0v0_Constantinople",
+				"callToEmptyThenCallError_d0g0v0_EIP158"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "stPreCompiledContracts",
+			"subtests": [
+				"identity_to_smaller_d0g0v0_Constantinople",
+				"identity_to_bigger_d0g0v0_Constantinople"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "stReturnDataTest",
+			"subtests": [
+				"modexp_modsize0_returndatasize_d0g1v0_Constantinople",
+				"modexp_modsize0_returndatasize_d0g2v0_Constantinople",
+				"modexp_modsize0_returndatasize_d0g3v0_Constantinople"
+			]
+		},
+		{
+			"reference": "9590",
+			"failing": "stSpecialTest",
+			"subtests": [
+				"push32withoutByte_d0g0v0_Constantinople"
+			]
+		}
+
+	],
+	"state":
+	[
+		{
+			"reference": "9590",
+			"failing": "stCreateTest",
+			"subtests": {
+				"CreateOOGafterInitCodeReturndata2": {
+					"subnumbers": ["2"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stCreate2Test",
+			"subtests": {
+				"RevertInCreateInInit": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stEIP150Specific",
+			"subtests": {
+				"NewGasPriceForCodes": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stInitCodeTest",
+			"subtests": {
+				"OutOfGasContractCreation": {
+					"subnumbers": ["4"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stPreCompiledContracts",
+			"subtests": {
+				"modexp": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stRevertTest",
+			"subtests": {
+				"LoopCallsDepthThenRevert3": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertOpcodeCreate": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertSubCallStorageOOG2": {
+					"subnumbers": ["1","3"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertDepthCreateOOG": {
+					"subnumbers": ["3","4"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertOpcodeMultipleSubCalls": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertOpcodeDirectCall": {
+					"subnumbers": ["1","2"],
+					"chain": "Constantinople (test)"
+				},
+				"LoopCallsDepthThenRevert2": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertDepth2": {
+					"subnumbers": ["1"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertRemoteSubCallStorageOOG2": {
+					"subnumbers": ["1","2"],
+					"chain": "Constantinople (test)"
+				},
+				"RevertDepthCreateAddressCollision": {
+					"subnumbers": ["3","4"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "9590",
+			"failing": "stStaticCall",
+			"subtests": {
+				"static_RevertDepth2": {
+					"subnumbers": ["1","3"],
+					"chain": "Constantinople (test)"
+				},
+				"static_CheckOpcodes4": {
+					"subnumbers": ["3"],
+					"chain": "Constantinople (test)"
+				},
+				"static_CheckOpcodes3": {
+					"subnumbers": ["5","6","7","8"],
+					"chain": "Constantinople (test)"
+				},
+				"static_callBasic": {
+					"subnumbers": ["1","2"],
+					"chain": "Constantinople (test)"
+				},
+				"static_CheckOpcodes2": {
+					"subnumbers": ["5","6","7","8"],
+					"chain": "Constantinople (test)"
+				},
+				"static_callCreate": {
+					"subnumbers": ["2"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		},
+		{
+			"reference": "https://github.com/ethereum/tests/issues/512",
+			"failing": "stZeroKnowledge",
+			"subtests": {
+				"pointAddTrunc": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				},
+				"pointAdd": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				},
+				"pointMulAdd": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				},
+				"pointMulAdd2": {
+					"subnumbers": ["*"],
+					"chain": "Constantinople (test)"
+				}
+			}
+		}
+	]
+}

ethcore/src/client/evm_test_client.rs

@@ -86,9 +86,9 @@ impl<'a> EvmTestClient<'a> {
 			ForkSpec::EIP150 => Some(ethereum::new_eip150_test()),
 			ForkSpec::EIP158 => Some(ethereum::new_eip161_test()),
 			ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()),
+			ForkSpec::Constantinople => Some(ethereum::new_constantinople_test()),
 			ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()),
 			ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None,
-			_ => None,
 		}
 	}
 

ethcore/src/json_tests/chain.rs

@@ -23,6 +23,7 @@ use miner::Miner;
 use io::IoChannel;
 use test_helpers;
 use verification::queue::kind::blocks::Unverified;
+use super::SKIP_TEST_STATE;
 
 use super::HookType;
 
@@ -36,12 +37,20 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(name: &String) -> bool {
+	SKIP_TEST_STATE.block.iter().any(|block_test|block_test.subtests.contains(name))
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::blockchain::Test::load(json_data).unwrap();
 	let mut failed = Vec::new();
 
 	for (name, blockchain) in tests.into_iter() {
+		if skip_test(&name) {
+			println!("   - {} | {:?} Ignoring tests because in skip list", name, blockchain.network);
+			continue;
+		}
 		start_stop_hook(&name, HookType::OnStart);
 
 		let mut fail = false;
@@ -122,19 +131,24 @@ mod block_tests {
 	declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"}
 	declare_test!{BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"}
 	declare_test!{BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"}
+	declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"}
 	declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"}
 	declare_test!{BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"}
 	declare_test!{BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"}
 	declare_test!{BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"}
 	declare_test!{BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"}
 
+	declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"}
@@ -149,12 +163,15 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"}
 	declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"}
@@ -166,6 +183,7 @@ mod block_tests {
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"}
 	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"}
+	declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"}
 
 	declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"}
 	declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"}

ethcore/src/json_tests/mod.rs

@@ -24,10 +24,12 @@ mod executive;
 mod state;
 mod chain;
 mod trie;
+mod skip;
 
 #[cfg(test)]
 mod difficulty;
 
+
 pub use self::test_common::HookType;
 
 pub use self::transaction::run_test_path as run_transaction_test_path;
@@ -42,3 +44,4 @@ pub use self::trie::run_generic_test_path as run_generic_trie_test_path;
 pub use self::trie::run_generic_test_file as run_generic_trie_test_file;
 pub use self::trie::run_secure_test_path as run_secure_trie_test_path;
 pub use self::trie::run_secure_test_file as run_secure_trie_test_file;
+use self::skip::SKIP_TEST_STATE;

ethcore/src/json_tests/skip.rs

@@ -0,0 +1,37 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State tests to skip.
+
+use ethjson;
+
+#[cfg(all(not(test), feature = "ci-skip-tests"))]
+compile_error!("ci-skip-tests can only be enabled for testing builds.");
+
+#[cfg(feature="ci-skip-issue")]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json");
+		ethjson::test::SkipStates::load(&skip_data[..]).expect("No invalid json allowed")
+	};
+}
+
+#[cfg(not(feature="ci-skip-issue"))]
+lazy_static!{
+	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
+		ethjson::test::SkipStates::empty()
+	};
+}

ethcore/src/json_tests/state.rs

@@ -22,7 +22,7 @@ use client::{EvmTestClient, EvmTestError, TransactResult};
 use ethjson;
 use transaction::SignedTransaction;
 use vm::EnvInfo;
-
+use super::SKIP_TEST_STATE;
 use super::HookType;
 
 /// Run state jsontests on a given folder.
@@ -35,6 +35,18 @@ pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
 	::json_tests::test_common::run_test_file(p, json_chain_test, h)
 }
 
+fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
+	SKIP_TEST_STATE.state.iter().any(|state_test|{
+		if let Some(subtest) = state_test.subtests.get(subname) {
+			chain == &subtest.chain &&
+			(subtest.subnumbers[0] == "*"
+				 || subtest.subnumbers.contains(&number.to_string()))
+		} else {
+			false
+		}
+	})
+}
+
 pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String> {
 	::ethcore_logger::init_log();
 	let tests = ethjson::state::test::Test::load(json_data).unwrap();
@@ -60,6 +72,10 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(json_data: &[u8], start_stop_ho
 
 				for (i, state) in states.into_iter().enumerate() {
 					let info = format!("   - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total);
+					if skip_test(&name, &spec.name, i + 1) {
+						println!("{} in skip list : SKIPPED", info);
+						continue;
+					}
 
 					let post_root: H256 = state.hash.into();
 					let transaction: SignedTransaction = multitransaction.select(&state.indexes).into();
@@ -125,11 +141,13 @@ mod state_tests {
 	declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"}
 	declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"}
 	declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"}
+	declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"}
 	declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"}
 	declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"}
 	declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"}
 	declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"}
 	declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"}
+	declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"}
 	declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"}
 	declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"}
 	declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"}

ethcore/src/json_tests/test_common.rs

@@ -34,6 +34,18 @@ pub fn run_test_path<H: FnMut(&str, HookType)>(
 	p: &Path, skip: &[&'static str],
 	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
 	start_stop_hook: &mut H
+) {
+	let mut errors = Vec::new();
+	run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors);
+	let empty: [String; 0] = [];
+	assert_eq!(errors, empty);
+}
+
+fn run_test_path_inner<H: FnMut(&str, HookType)>(
+	p: &Path, skip: &[&'static str],
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H,
+	errors: &mut Vec<String>
 ) {
 	let path = Path::new(p);
 	let s: HashSet<OsString> = skip.iter().map(|s| {
@@ -50,17 +62,32 @@ pub fn run_test_path<H: FnMut(&str, HookType)>(
 			} else {
 				Some(e.path())
 			}}) {
-			run_test_path(&p, skip, runner, start_stop_hook)
+			run_test_path_inner(&p, skip, runner, start_stop_hook, errors);
 		}
 	} else if extension == Some("swp") || extension == None {
 		// Ignore junk
 	} else {
 		let mut path = p.to_path_buf();
 		path.set_extension("json");
-		run_test_file(&path, runner, start_stop_hook)
+		run_test_file_append(&path, runner, start_stop_hook, errors)
 	}
 }
 
+fn run_test_file_append<H: FnMut(&str, HookType)>(
+	path: &Path,
+	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
+	start_stop_hook: &mut H,
+	errors: &mut Vec<String>
+) {
+	let mut data = Vec::new();
+	let mut file = match File::open(&path) {
+		Ok(file) => file,
+		Err(_) => panic!("Error opening test file at: {:?}", path),
+	};
+	file.read_to_end(&mut data).expect("Error reading test file");
+	errors.append(&mut runner(&data, start_stop_hook));
+}
+
 pub fn run_test_file<H: FnMut(&str, HookType)>(
 	path: &Path,
 	runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,

json/src/test/mod.rs

@@ -45,6 +45,7 @@ pub struct DifficultyTestCase {
 	#[serde(rename="currentBlockNumber")]
 	pub current_block_number: Uint,
 }
+
 /// Blockchain test deserializer.
 #[derive(Debug, PartialEq, Deserialize)]
 pub struct DifficultyTest(BTreeMap<String, DifficultyTestCase>);
@@ -64,3 +65,59 @@ impl DifficultyTest {
 		serde_json::from_reader(reader)
 	}
 }
+
+/// Test to skip (only if issue ongoing)
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct SkipStates {
+	/// Block tests
+	pub block: Vec<BlockSkipStates>,
+	/// State tests
+	pub state: Vec<StateSkipStates>,
+
+}
+
+/// Block test to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct BlockSkipStates {
+	/// Issue reference.
+	pub reference: String,
+	/// Test failing name.
+	pub failing: String,
+	/// Items failing for the test.
+	pub subtests: Vec<String>,
+}
+
+/// State test to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct StateSkipStates {
+	/// Issue reference.
+	pub reference: String,
+	/// Test failing name.
+	pub failing: String,
+	/// Items failing for the test.
+	pub subtests: BTreeMap<String, StateSkipSubStates>
+}
+
+/// State subtest to skip.
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct StateSkipSubStates {
+	/// State test number of this item. Or '*' for all state.
+	pub subnumbers: Vec<String>,
+	/// Chain for this items.
+	pub chain: String,
+}
+
+impl SkipStates {
+	/// Loads skip states from json.
+	pub fn load<R>(reader: R) -> Result<Self, Error> where R: Read {
+		serde_json::from_reader(reader)
+	}
+
+	/// Empty skip states.
+	pub fn empty() -> Self {
+		SkipStates {
+			block: Vec::new(),
+			state: Vec::new(),
+		}
+	}
+}

test.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Running Parity Full Test Suite
 
-FEATURES="json-tests"
+FEATURES="json-tests,ci-skip-issue"
 OPTIONS="--release"
 VALIDATE=1