diff --git a/Cargo.lock b/Cargo.lock index 5a25ec4e3..c1d3ec145 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,6 +830,7 @@ dependencies = [ "ethkey 0.3.0", "evm 0.1.0", "fetch 0.1.0", + "globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -861,8 +862,10 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "time-utils 0.1.0", "trace-time 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -871,6 +874,7 @@ dependencies = [ "unexpected 0.1.0", "using_queue 0.1.0", "vm 0.1.0", + "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "wasm 0.1.0", ] @@ -3865,6 +3869,19 @@ dependencies = [ "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "term_size" version = "0.3.1" @@ -4984,6 +5001,7 @@ dependencies = [ "checksum synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "575be94ccb86e8da37efb894a87e2b660be299b41d8ef347f9d6d79fbe61b1ba" "checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" "checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 7569c6a9a..38df20e10 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -31,6 +31,7 @@ ethereum-types = "0.4" ethjson = { path = "../json" } ethkey = { path = "../accounts/ethkey" } evm = { path = "evm" } +globset = "0.4" hash-db = "0.11.0" heapsize = "0.4" itertools = "0.5" @@ -60,14 +61,17 @@ rlp_derive = { path = "../util/rlp-derive" } rustc-hex = "1.0" serde = "1.0" serde_derive = "1.0" +serde_json = "1.0" stats = { path = "../util/stats" } tempdir = {version="0.3", optional = true} +tempfile = "3.1.0" time-utils = { path = "../util/time-utils" } trace-time = "0.1" triehash-ethereum = { version = "0.2", path = "../util/triehash-ethereum" } unexpected = { path = "../util/unexpected" } using_queue = { path = "../miner/using-queue" } vm = { path = "vm" } +walkdir = "2" wasm = { path = "wasm" } [dev-dependencies] diff --git a/ethcore/res/ethereum/runner/full.json b/ethcore/res/ethereum/runner/full.json new file mode 100644 index 000000000..2d7e98a2a --- /dev/null +++ b/ethcore/res/ethereum/runner/full.json @@ -0,0 +1,51 @@ +{ + "chain": [ + { + "path": "res/ethereum/tests/BlockchainTests", + "skip" : [] + } + ], + "state": [ + { + "path": "res/ethereum/tests/GeneralStateTests", + "skip" : [] + + } + ], + "difficulty": [ + { + "path": [ + "res/ethereum/tests/BasicTests/difficulty.json", + "res/ethereum/tests/BasicTests/difficultyMainNetwork.json" + ], + "chainspec": "Foundation" + } + ], + "executive": [ + { + "path": "res/ethereum/tests/VMTests" + } + ], + "transaction": [ + { + "path": "res/ethereum/tests/TransactionTests" + } + ], + "trie": [ + { + "path": [ + "res/ethereum/tests/TrieTests/trietest.json", + "res/ethereum/tests/TrieTests/trieanyorder.json" + ], + "triespec": "Generic" + }, + { + "path": [ + "res/ethereum/tests/TrieTests/hex_encoded_securetrie_test.json", + "res/ethereum/tests/TrieTests/trietest_secureTrie.json", + "res/ethereum/tests/TrieTests/trieanyorder_secureTrie.json" + ], + "triespec": "Secure" + } + ] +} \ No newline at end of file diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index b331a5bcb..256f12c87 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -107,6 +107,7 @@ impl<'a> EvmTestClient<'a> { ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, + ForkSpec::ByzantiumToConstantinopleAt5 => None, } } diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index f95f1deea..decbf9370 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -14,50 +14,135 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use super::{HookType, SKIP_TEST_STATE}; -use client::{ChainInfo, Client, ClientConfig, EvmTestClient, ImportBlock}; +use super::HookType; +use client::{ + Balance, BlockChainClient, BlockId, ChainInfo, Client, ClientConfig, EvmTestClient, + ImportBlock, Nonce, StateOrBlock, +}; +use ethereum_types::{H256, U256}; use ethjson; use io::IoChannel; +use log::warn; use miner::Miner; +use rustc_hex::ToHex; use spec::Genesis; use std::{path::Path, sync::Arc}; use test_helpers; use verification::{queue::kind::blocks::Unverified, VerifierType}; -/// Run chain jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) -} +fn check_poststate( + client: &Arc, + test_name: &str, + post_state: ethjson::blockchain::State, +) -> bool { + let mut success = true; -/// Run chain jsontests on a given file. -pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, json_chain_test, h) -} + for (address, expected) in post_state { + if let Some(expected_balance) = expected.balance { + let expected_balance: U256 = expected_balance.into(); + let current_balance = client + .balance( + &address.clone().into(), + StateOrBlock::Block(BlockId::Latest), + ) + .unwrap(); + if expected_balance != current_balance { + warn!(target: "json-tests", "{} – Poststate {:?} balance mismatch current={} expected={}", + test_name, address, current_balance, expected_balance); + success = false; + } + } -fn skip_test(name: &String) -> bool { - SKIP_TEST_STATE - .block - .iter() - .any(|block_test| block_test.subtests.contains(name)) + if let Some(expected_nonce) = expected.nonce { + let expected_nonce: U256 = expected_nonce.into(); + let current_nonce = client + .nonce(&address.clone().into(), BlockId::Latest) + .unwrap(); + if expected_nonce != current_nonce { + warn!(target: "json-tests", "{} – Poststate {:?} nonce mismatch current={} expected={}", + test_name, address, current_nonce, expected_nonce); + success = false; + } + } + + if let Some(expected_code) = expected.code { + let expected_code: String = expected_code.to_hex(); + let current_code = match client.code( + &address.clone().into(), + StateOrBlock::Block(BlockId::Latest), + ) { + Some(Some(code)) => code.to_hex(), + _ => "".to_string(), + }; + if current_code != expected_code { + warn!(target: "json-tests", "{} – Poststate {:?} code mismatch current={} expected={}", + test_name, address, current_code, expected_code); + success = false; + } + } + + if let Some(expected_storage) = expected.storage { + for (uint_position, uint_expected_value) in expected_storage.iter() { + let mut position = H256::default(); + uint_position.0.to_big_endian(position.as_mut()); + + let mut expected_value = H256::default(); + uint_expected_value.0.to_big_endian(expected_value.as_mut()); + + let current_value = client + .storage_at( + &address.clone().into(), + &position, + StateOrBlock::Block(BlockId::Latest), + ) + .unwrap(); + + if current_value != expected_value { + let position: &[u8] = position.as_ref(); + let current_value: &[u8] = current_value.as_ref(); + let expected_value: &[u8] = expected_value.as_ref(); + warn!(target: "json-tests", "{} – Poststate {:?} state {} mismatch actual={} expected={}", + test_name, address, position.to_hex(), current_value.to_hex(), + expected_value.to_hex()); + success = false; + } + } + } + + if expected.builtin.is_some() { + warn!(target: "json-tests", "{} – Poststate {:?} builtin not supported", test_name, address); + success = false; + } + if expected.constructor.is_some() { + warn!(target: "json-tests", "{} – Poststate {:?} constructor not supported", test_name, address); + success = false; + } + } + success } pub fn json_chain_test( + test: ðjson::test::ChainTests, + path: &Path, json_data: &[u8], start_stop_hook: &mut H, ) -> Vec { let _ = ::env_logger::try_init(); - let tests = ethjson::blockchain::Test::load(json_data).unwrap(); + let tests = ethjson::blockchain::Test::load(json_data).expect(&format!( + "Could not parse JSON chain test data from {}", + path.display() + )); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { - if skip_test(&name) { - println!( - " - {} | {:?} Ignoring tests because in skip list", - name, blockchain.network - ); + let skip_test = test + .skip + .iter() + .any(|block_test| block_test.names.contains(&name)); + if skip_test { + info!(" SKIPPED {:?} {:?}", name, blockchain.network); continue; } - start_stop_hook(&name, HookType::OnStart); let mut fail = false; { @@ -72,14 +157,12 @@ pub fn json_chain_test( } }; - flush!(" - {}...", name); - let spec = { let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) { Some(spec) => spec, None => { - println!( - " - {} | {:?} Ignoring tests because of missing spec", + info!( + " SKIPPED {:?} {:?} - Unimplemented chainspec ", name, blockchain.network ); continue; @@ -91,10 +174,11 @@ pub fn json_chain_test( spec.set_genesis_state(state) .expect("Failed to overwrite genesis state"); spec.overwrite_genesis_params(genesis); - assert!(spec.is_state_root_valid()); spec }; + start_stop_hook(&name, HookType::OnStart); + { let db = test_helpers::new_db(); let mut config = ClientConfig::default(); @@ -103,6 +187,7 @@ pub fn json_chain_test( config.check_seal = false; } config.history = 8; + config.queue.verifier_settings.num_verifiers = 1; let client = Client::new( config, &spec, @@ -110,103 +195,49 @@ pub fn json_chain_test( Arc::new(Miner::new_for_tests(&spec, None)), IoChannel::disconnected(), ) - .unwrap(); + .expect("Failed to instantiate a new Client"); + for b in blockchain.blocks_rlp() { - if let Ok(block) = Unverified::from_rlp(b) { - let _ = client.import_block(block); - client.flush_queue(); - client.import_verified_blocks(); + let bytes_len = b.len(); + let block = Unverified::from_rlp(b); + match block { + Ok(block) => { + let num = block.header.number(); + trace!(target: "json-tests", "{} – Importing {} bytes. Block #{}", name, bytes_len, num); + let res = client.import_block(block); + if let Err(e) = res { + warn!(target: "json-tests", "{} – Error importing block #{}: {:?}", name, num, e); + } + client.flush_queue(); + client.import_verified_blocks(); + } + Err(decoder_err) => { + warn!(target: "json-tests", "Error decoding test block: {:?} ({} bytes)", decoder_err, bytes_len); + } } } - fail_unless(client.chain_info().best_block_hash == blockchain.best_block.into()); + + let post_state_success = if let Some(post_state) = blockchain.post_state.clone() { + check_poststate(&client, &name, post_state) + } else { + true + }; + + fail_unless( + client.chain_info().best_block_hash == blockchain.best_block.into() + && post_state_success, + ); } } - if !fail { - flushln!("ok"); + if fail { + flushln!(" - chain: {}...FAILED", name); + } else { + flushln!(" - chain: {}...OK", name); } start_stop_hook(&name, HookType::OnStop); } - println!("!!! {:?} tests from failed.", failed.len()); failed } - -#[cfg(test)] -mod block_tests { - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(json_data, h) - } - - declare_test! {BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} - declare_test! {BlockchainTests_bcExploitTest, "BlockchainTests/bcExploitTest"} - declare_test! {BlockchainTests_bcForgedTest, "BlockchainTests/bcForgedTest"} - declare_test! {BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} - declare_test! {BlockchainTests_bcGasPricerTest, "BlockchainTests/bcGasPricerTest"} - declare_test! {BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} - declare_test! {BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} - declare_test! {BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"} - declare_test! {BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"} - declare_test! {BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} - declare_test! {BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"} - declare_test! {BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} - declare_test! {BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} - declare_test! {BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} - - declare_test! {BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"} - declare_test! {BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"} - declare_test! {BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"} - declare_test! {BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"} - declare_test! {BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test! {BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test! {BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"} - declare_test! {BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"} - declare_test! {BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"} - declare_test! {BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test! {BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test! {BlockchainTests_GeneralStateTest_stEIP150Specific, "BlockchainTests/GeneralStateTests/stEIP150Specific/"} - declare_test! {BlockchainTests_GeneralStateTest_stEIP158Specific, "BlockchainTests/GeneralStateTests/stEIP158Specific/"} - declare_test! {BlockchainTests_GeneralStateTest_stExample, "BlockchainTests/GeneralStateTests/stExample/"} - declare_test! {BlockchainTests_GeneralStateTest_stHomesteadSpecific, "BlockchainTests/GeneralStateTests/stHomesteadSpecific/"} - declare_test! {BlockchainTests_GeneralStateTest_stInitCodeTest, "BlockchainTests/GeneralStateTests/stInitCodeTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stLogTests, "BlockchainTests/GeneralStateTests/stLogTests/"} - declare_test! {BlockchainTests_GeneralStateTest_stMemExpandingEIP150Calls, "BlockchainTests/GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test! {heavy => BlockchainTests_GeneralStateTest_stMemoryStressTest, "BlockchainTests/GeneralStateTests/stMemoryStressTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"} - declare_test! {BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"} - declare_test! {heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"} - declare_test! {BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"} - declare_test! {BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"} - declare_test! {BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"} - declare_test! {BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"} - declare_test! {BlockchainTests_GeneralStateTest_stStaticCall, "BlockchainTests/GeneralStateTests/stStaticCall/"} - declare_test! {BlockchainTests_GeneralStateTest_stSystemOperationsTest, "BlockchainTests/GeneralStateTests/stSystemOperationsTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stTransactionTest, "BlockchainTests/GeneralStateTests/stTransactionTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stTransitionTest, "BlockchainTests/GeneralStateTests/stTransitionTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stWalletTest, "BlockchainTests/GeneralStateTests/stWalletTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"} - declare_test! {BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"} - declare_test! {BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"} - declare_test! {BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"} - declare_test! {BlockchainTests_GeneralStateTest_stSStoreTest, "BlockchainTests/GeneralStateTests/stSStoreTest/"} - - declare_test! {BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"} - declare_test! {BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"} - declare_test! {BlockchainTests_TransitionTests_bcHomesteadToDao, "BlockchainTests/TransitionTests/bcHomesteadToDao/"} - declare_test! {BlockchainTests_TransitionTests_bcHomesteadToEIP150, "BlockchainTests/TransitionTests/bcHomesteadToEIP150/"} -} diff --git a/ethcore/src/json_tests/difficulty.rs b/ethcore/src/json_tests/difficulty.rs index fc47a2768..53b432e73 100644 --- a/ethcore/src/json_tests/difficulty.rs +++ b/ethcore/src/json_tests/difficulty.rs @@ -17,25 +17,28 @@ use ethereum_types::U256; use ethjson; use spec::Spec; +use std::path::Path; use types::header::Header; use super::HookType; pub fn json_difficulty_test( + path: &Path, json_data: &[u8], spec: Spec, start_stop_hook: &mut H, ) -> Vec { - let _ = ::env_logger::try_init(); - let tests = ethjson::test::DifficultyTest::load(json_data).unwrap(); + let mut ret = Vec::new(); + let _ = env_logger::try_init(); + let tests = ethjson::test::DifficultyTest::load(json_data).expect(&format!( + "Could not parse JSON difficulty test data from {}", + path.display() + )); let engine = &spec.engine; for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); - flush!(" - {}...", name); - println!(" - {}...", name); - let mut parent_header = Header::new(); let block_number: u64 = test.current_block_number.into(); parent_header.set_number(block_number - 1); @@ -48,65 +51,14 @@ pub fn json_difficulty_test( header.set_timestamp(test.current_timestamp.into()); engine.populate_from_parent(&mut header, &parent_header); let expected_difficulty: U256 = test.current_difficulty.into(); - assert_eq!(header.difficulty(), &expected_difficulty); - flushln!("ok"); + if header.difficulty() == &expected_difficulty { + flushln!(" - difficulty: {}...OK", name); + } else { + flushln!(" - difficulty: {}...FAILED", name); + ret.push(format!("{}:{}", path.to_string_lossy(), name)); + } start_stop_hook(&name, HookType::OnStop); } - vec![] -} - -macro_rules! difficulty_json_test { - ( $spec:ident ) => { - use super::json_difficulty_test; - use json_tests::HookType; - use tempdir::TempDir; - - fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { - let tempdir = TempDir::new("").unwrap(); - json_difficulty_test(json_data, ::ethereum::$spec(&tempdir.path()), h) - } - }; -} - -macro_rules! difficulty_json_test_nopath { - ( $spec:ident ) => { - use super::json_difficulty_test; - use json_tests::HookType; - - fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { - json_difficulty_test(json_data, ::ethereum::$spec(), h) - } - }; -} - -mod difficulty_test { - difficulty_json_test!(new_foundation); - declare_test! {DifficultyTests_difficulty, "BasicTests/difficulty.json"} -} - -mod difficulty_test_byzantium { - difficulty_json_test_nopath!(new_byzantium_test); - declare_test! {DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"} -} - -mod difficulty_test_foundation { - difficulty_json_test!(new_foundation); - declare_test! {DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"} -} - -// Disabling Ropsten diff tests; waiting for upstream ethereum/tests Constantinople update -//mod difficulty_test_ropsten { -// difficulty_json_test_nopath!(new_ropsten_test); -// declare_test!{DifficultyTests_difficultyRopsten, "BasicTests/difficultyRopsten.json"} -//} - -mod difficulty_test_frontier { - difficulty_json_test_nopath!(new_frontier_test); - declare_test! {DifficultyTests_difficultyFrontier, "BasicTests/difficultyFrontier.json"} -} - -mod difficulty_test_homestead { - difficulty_json_test_nopath!(new_homestead_test); - declare_test! {DifficultyTests_difficultyHomestead, "BasicTests/difficultyHomestead.json"} + ret } diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index f02c83e42..c65783cf9 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -18,7 +18,7 @@ use super::test_common::*; use bytes::Bytes; use ethjson; use ethtrie; -use evm::{Finalize, VMType}; +use evm::Finalize; use executive::*; use externalities::*; use hash::keccak; @@ -35,16 +35,6 @@ use vm::{ use super::HookType; -/// Run executive jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) -} - -/// Run executive jsontests on a given file. -pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, do_json_test, h) -} - #[derive(Debug, PartialEq, Clone)] struct CallCreate { data: Bytes, @@ -250,30 +240,25 @@ where } } -fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { - let vms = VMType::all(); - vms.iter() - .flat_map(|vm| do_json_test_for(vm, json_data, h)) - .collect() -} - -fn do_json_test_for( - vm_type: &VMType, +pub fn json_executive_test( + path: &Path, json_data: &[u8], start_stop_hook: &mut H, ) -> Vec { - let tests = ethjson::vm::Test::load(json_data).unwrap(); + let tests = ethjson::vm::Test::load(json_data).expect(&format!( + "Could not parse JSON executive test data from {}", + path.display() + )); let mut failed = Vec::new(); for (name, vm) in tests.into_iter() { - start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart); + start_stop_hook(&format!("{}", name), HookType::OnStart); - info!(target: "jsontests", "name: {:?}", name); let mut fail = false; let mut fail_unless = |cond: bool, s: &str| { if !cond && !fail { - failed.push(format!("[{}] {}: {}", vm_type, name, s)); + failed.push(format!("{}: {}", name, s)); fail = true } }; @@ -362,27 +347,35 @@ fn do_json_test_for( for (address, account) in vm.post_state.unwrap().into_iter() { let address = address.into(); - let code: Vec = account.code.into(); - let found_code = try_fail!(state.code(&address)); + if let Some(code) = account.code { + let code: Vec = code.into(); + let found_code = try_fail!(state.code(&address)); + fail_unless( + found_code + .as_ref() + .map_or_else(|| code.is_empty(), |c| &**c == &code), + "code is incorrect", + ); + } let found_balance = try_fail!(state.balance(&address)); let found_nonce = try_fail!(state.nonce(&address)); - - fail_unless( - found_code - .as_ref() - .map_or_else(|| code.is_empty(), |c| &**c == &code), - "code is incorrect", - ); - fail_unless( - found_balance == account.balance.into(), - "balance is incorrect", - ); - fail_unless(found_nonce == account.nonce.into(), "nonce is incorrect"); - for (k, v) in account.storage { - let key: U256 = k.into(); - let value: U256 = v.into(); - let found_storage = try_fail!(state.storage_at(&address, &From::from(key))); - fail_unless(found_storage == From::from(value), "storage is incorrect"); + if let Some(balance) = account.balance { + fail_unless(found_balance == balance.into(), "balance is incorrect"); + } + if let Some(nonce) = account.nonce { + fail_unless(found_nonce == nonce.into(), "nonce is incorrect"); + } + if let Some(storage) = account.storage { + for (k, v) in storage { + let key: U256 = k.into(); + let value: U256 = v.into(); + let found_storage = + try_fail!(state.storage_at(&address, &From::from(&key))); + fail_unless( + found_storage == From::from(&value), + "storage is incorrect", + ); + } } } @@ -392,26 +385,14 @@ fn do_json_test_for( } }; - start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop); - } + if fail { + println!(" - vm: {:?}...FAILED", name); + } else { + println!(" - vm: {:?}...OK", name); + } - for f in &failed { - error!("FAILED: {:?}", f); + start_stop_hook(&format!("{}", name), HookType::OnStop); } failed } - -declare_test! {ExecutiveTests_vmArithmeticTest, "VMTests/vmArithmeticTest"} -declare_test! {ExecutiveTests_vmBitwiseLogicOperationTest, "VMTests/vmBitwiseLogicOperation"} -declare_test! {ExecutiveTests_vmBlockInfoTest, "VMTests/vmBlockInfoTest"} -// TODO [todr] Fails with Signal 11 when using JIT -declare_test! {ExecutiveTests_vmEnvironmentalInfoTest, "VMTests/vmEnvironmentalInfo"} -declare_test! {ExecutiveTests_vmIOandFlowOperationsTest, "VMTests/vmIOandFlowOperations"} -declare_test! {ExecutiveTests_vmLogTest, "VMTests/vmLogTest"} -declare_test! {heavy => ExecutiveTests_vmPerformance, "VMTests/vmPerformance"} -declare_test! {ExecutiveTests_vmPushDupSwapTest, "VMTests/vmPushDupSwapTest"} -declare_test! {ExecutiveTests_vmRandomTest, "VMTests/vmRandomTest"} -declare_test! {ExecutiveTests_vmSha3Test, "VMTests/vmSha3Test"} -declare_test! {ExecutiveTests_vmSystemOperationsTest, "VMTests/vmSystemOperations"} -declare_test! {ExecutiveTests_vmTests, "VMTests/vmTests"} diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index 69572baaf..99fb33be5 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -16,35 +16,16 @@ //! Helpers and tests for operating on jsontests. -#[macro_use] -mod test_common; - mod chain; +mod difficulty; mod executive; -mod skip; +pub mod runner; mod state; +mod test_common; mod transaction; mod trie; -#[cfg(test)] -mod difficulty; - -pub use self::test_common::HookType; - -use self::skip::SKIP_TEST_STATE; pub use self::{ - chain::{run_test_file as run_chain_test_file, run_test_path as run_chain_test_path}, - executive::{ - run_test_file as run_executive_test_file, run_test_path as run_executive_test_path, - }, - state::{run_test_file as run_state_test_file, run_test_path as run_state_test_path}, - transaction::{ - run_test_file as run_transaction_test_file, run_test_path as run_transaction_test_path, - }, - trie::{ - run_generic_test_file as run_generic_trie_test_file, - run_generic_test_path as run_generic_trie_test_path, - run_secure_test_file as run_secure_trie_test_file, - run_secure_test_path as run_secure_trie_test_path, - }, + executive::json_executive_test, + test_common::{find_json_files_recursive, HookType}, }; diff --git a/ethcore/src/json_tests/runner.rs b/ethcore/src/json_tests/runner.rs new file mode 100644 index 000000000..283dec8b1 --- /dev/null +++ b/ethcore/src/json_tests/runner.rs @@ -0,0 +1,248 @@ +use ethjson::test::{ + ChainTests, DifficultyTests, EthereumTestSuite, ExecutiveTests, StateTests, TestChainSpec, + TestTrieSpec, TransactionTests, TrieTests, +}; +use globset::Glob; +use log::info; +use rayon::prelude::*; +use std::path::{Path, PathBuf}; +use tempfile::tempdir; +use trie::TrieSpec; + +/// Result of tests execution +pub struct TestResult { + /// Number of success execution + pub success: usize, + /// Number of success execution + pub failed: Vec, +} + +impl TestResult { + /// Creates a new TestResult without results + pub fn zero() -> Self { + TestResult { + success: 0, + failed: Vec::new(), + } + } + /// Creates a new success TestResult + pub fn success() -> Self { + TestResult { + success: 1, + failed: Vec::new(), + } + } + /// Creates a new failed TestResult + pub fn failed(name: &str) -> Self { + TestResult { + success: 0, + failed: vec![name.to_string()], + } + } +} + +impl std::ops::Add for TestResult { + type Output = Self; + + fn add(self, other: Self) -> Self { + let mut mself = self; + mself.success += other.success; + mself.failed.extend_from_slice(&other.failed); + mself + } +} + +impl std::ops::AddAssign for TestResult { + fn add_assign(&mut self, other: Self) { + self.success += other.success; + self.failed.extend_from_slice(&other.failed); + } +} + +pub struct TestRunner(EthereumTestSuite); + +impl TestRunner { + /// Loads a new JSON Test suite + pub fn load(reader: R) -> Result + where + R: std::io::Read, + { + Ok(TestRunner(serde_json::from_reader(reader)?)) + } + + /// Run the tests with one thread + pub fn run_without_par(&self) -> TestResult { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap(); + pool.install(|| self.run()) + } + + /// Run the tests + pub fn run(&self) -> TestResult { + let mut res = TestResult::zero(); + for t in &self.0.chain { + res += Self::run_chain_tests(&t); + } + for t in &self.0.state { + res += Self::run_state_tests(&t); + } + for t in &self.0.difficulty { + res += Self::run_difficuly_tests(&t); + } + for t in &self.0.executive { + res += Self::run_executive_tests(&t); + } + for t in &self.0.transaction { + res += Self::run_transaction_tests(&t); + } + for t in &self.0.trie { + res += Self::run_trie_tests(&t); + } + res + } + + fn run1(test: &T, base_path: &PathBuf, f: F) -> TestResult + where + T: Send + Sync, + F: Fn(&T, &Path, &[u8]) -> Vec + Send + Sync, + { + let result = super::find_json_files_recursive(&base_path) + .into_par_iter() + .map(|path| { + info!("{:?}", path); + let json = std::fs::read(&path).unwrap(); + let faileds = f(test, &path, &json); + if faileds.len() > 0 { + TestResult::failed(&faileds.join(",")) + } else { + TestResult::success() + } + }) + .reduce(TestResult::zero, |a, b| a + b); + + if result.success + result.failed.len() == 0 { + panic!("There is no tests in the specified path {:?}", base_path); + } + result + } + + fn in_set(path: &Path, exprs: &[String]) -> bool { + for pathexp in exprs { + let glob = Glob::new(&pathexp) + .expect(&format!("cannot parse expression {}", pathexp)) + .compile_matcher(); + if glob.is_match(path) { + return true; + } + } + false + } + + fn run_chain_tests(test: &ChainTests) -> TestResult { + Self::run1( + test, + &test.path, + |test: &ChainTests, path: &Path, json: &[u8]| { + for skip in &test.skip { + if Self::in_set(&path, &skip.paths) { + println!(" - {} ..SKIPPED", path.to_string_lossy()); + return Vec::new(); + } + } + super::chain::json_chain_test(&test, &path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_state_tests(test: &StateTests) -> TestResult { + Self::run1( + test, + &test.path, + |test: &StateTests, path: &Path, json: &[u8]| { + for skip in &test.skip { + if Self::in_set(&path, &skip.paths) { + println!(" - {} ..SKIPPED", path.to_string_lossy()); + return Vec::new(); + } + } + super::state::json_chain_test(&test, &path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_difficuly_tests(test: &DifficultyTests) -> TestResult { + let mut acc = TestResult::zero(); + for path in &test.path { + acc += Self::run1( + test, + &path, + |test: &DifficultyTests, path: &Path, json: &[u8]| { + let spec = match &test.chainspec { + TestChainSpec::Foundation => { + crate::ethereum::new_foundation(&tempdir().unwrap().path()) + } + TestChainSpec::ByzantiumTest => crate::ethereum::new_byzantium_test(), + TestChainSpec::FrontierTest => crate::ethereum::new_frontier_test(), + TestChainSpec::HomesteadTest => crate::ethereum::new_homestead_test(), + }; + super::difficulty::json_difficulty_test(&path, &json, spec, &mut |_, _| {}) + }, + ) + } + acc + } + + fn run_executive_tests(test: &ExecutiveTests) -> TestResult { + Self::run1( + test, + &test.path, + |_: &ExecutiveTests, path: &Path, json: &[u8]| { + super::executive::json_executive_test(&path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_transaction_tests(test: &TransactionTests) -> TestResult { + Self::run1( + test, + &test.path, + |_: &TransactionTests, path: &Path, json: &[u8]| { + super::transaction::json_transaction_test(&path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_trie_tests(test: &TrieTests) -> TestResult { + let mut acc = TestResult::zero(); + for path in &test.path { + acc += Self::run1(test, &path, |test: &TrieTests, path: &Path, json: &[u8]| { + let spec = match &test.triespec { + TestTrieSpec::Generic => TrieSpec::Generic, + TestTrieSpec::Secure => TrieSpec::Secure, + }; + super::trie::json_trie_test(&path, &json, spec, &mut |_, _| {}) + }); + } + acc + } +} + +#[test] +fn ethereum_json_tests() { + let content = std::fs::read("res/ethereum/runner/full.json") + .expect("cannot open ethereum tests spec file"); + let runner = + TestRunner::load(content.as_slice()).expect("cannot load ethereum tests spec file"); + println!("----------------------------------------------------"); + let result = runner.run(); + println!("----------------------------------------------------"); + flushln!( + "SUCCESS: {} FAILED: {} {:?}", + result.success, + result.failed.len(), + result.failed + ); + assert!(result.failed.len() == 0); +} diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 302fa5d71..8308926f6 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -use super::{test_common::*, HookType, SKIP_TEST_STATE}; +use super::{test_common::*, HookType}; use client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; use ethjson; use pod_state::PodState; @@ -23,19 +23,16 @@ use trace; use types::transaction::SignedTransaction; use vm::EnvInfo; -/// Run state jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) -} - -/// Run state jsontests on a given file. -pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, json_chain_test, h) -} - -fn skip_test(subname: &str, chain: &String, number: usize) -> bool { - SKIP_TEST_STATE.state.iter().any(|state_test| { - if let Some(subtest) = state_test.subtests.get(subname) { +fn skip_test( + test: ðjson::test::StateTests, + subname: &str, + chain: &String, + number: usize, +) -> bool { + trace!(target: "json-tests", "[state, skip_test] subname: '{}', chain: '{}', number: {}", subname, chain, number); + test.skip.iter().any(|state_test| { + if let Some(subtest) = state_test.names.get(subname) { + trace!(target: "json-tests", "[state, skip_test] Maybe skipping {:?}", subtest); chain == &subtest.chain && (subtest.subnumbers[0] == "*" || subtest.subnumbers.contains(&number.to_string())) @@ -46,11 +43,16 @@ fn skip_test(subname: &str, chain: &String, number: usize) -> bool { } pub fn json_chain_test( + state_test: ðjson::test::StateTests, + path: &Path, json_data: &[u8], start_stop_hook: &mut H, ) -> Vec { let _ = ::env_logger::try_init(); - let tests = ethjson::state::test::Test::load(json_data).unwrap(); + let tests = ethjson::state::test::Test::load(json_data).expect(&format!( + "Could not parse JSON state test data from {}", + path.display() + )); let mut failed = Vec::new(); for (name, test) in tests.into_iter() { @@ -66,18 +68,23 @@ pub fn json_chain_test( let spec = match EvmTestClient::spec_from_json(&spec_name) { Some(spec) => spec, None => { - println!( - " - {} | {:?} Ignoring tests because of missing spec", - name, spec_name + panic!( + "Unimplemented chainspec '{:?}' in test '{}'", + spec_name, name ); - continue; } }; for (i, state) in states.into_iter().enumerate() { - let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); - if skip_test(&name, &spec.name, i + 1) { - println!("{} in skip list : SKIPPED", info); + let info = format!( + " - state: {} | {:?} ({}/{}) ...", + name, + spec_name, + i + 1, + total + ); + if skip_test(&state_test, &name, &spec.name, i + 1) { + println!("{}: SKIPPED", info); continue; } @@ -134,74 +141,5 @@ pub fn json_chain_test( start_stop_hook(&name, HookType::OnStop); } - if !failed.is_empty() { - println!("!!! {:?} tests failed.", failed.len()); - } failed } - -#[cfg(test)] -mod state_tests { - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(json_data, h) - } - - declare_test! {GeneralStateTest_stArgsZeroOneBalance, "GeneralStateTests/stArgsZeroOneBalance/"} - declare_test! {GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"} - declare_test! {GeneralStateTest_stBadOpcodeTest, "GeneralStateTests/stBadOpcode/"} - declare_test! {GeneralStateTest_stBugs, "GeneralStateTests/stBugs/"} - declare_test! {GeneralStateTest_stCallCodes, "GeneralStateTests/stCallCodes/"} - declare_test! {GeneralStateTest_stCallCreateCallCodeTest, "GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test! {GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test! {GeneralStateTest_stCallDelegateCodesHomestead, "GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test! {GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"} - declare_test! {GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"} - declare_test! {GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"} - declare_test! {GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"} - declare_test! {GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"} - declare_test! {GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test! {GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test! {GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"} - declare_test! {GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"} - declare_test! {GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"} - declare_test! {GeneralStateTest_stExample, "GeneralStateTests/stExample/"} - declare_test! {GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"} - declare_test! {GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"} - declare_test! {GeneralStateTest_stLogTests, "GeneralStateTests/stLogTests/"} - declare_test! {GeneralStateTest_stMemExpandingEIP150Calls, "GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test! {heavy => GeneralStateTest_stMemoryStressTest, "GeneralStateTests/stMemoryStressTest/"} - declare_test! {GeneralStateTest_stMemoryTest, "GeneralStateTests/stMemoryTest/"} - declare_test! {GeneralStateTest_stNonZeroCallsTest, "GeneralStateTests/stNonZeroCallsTest/"} - declare_test! {GeneralStateTest_stPreCompiledContracts, "GeneralStateTests/stPreCompiledContracts/"} - declare_test! {GeneralStateTest_stPreCompiledContracts2, "GeneralStateTests/stPreCompiledContracts2/"} - declare_test! {heavy => GeneralStateTest_stQuadraticComplexityTest, "GeneralStateTests/stQuadraticComplexityTest/"} - declare_test! {GeneralStateTest_stRandom, "GeneralStateTests/stRandom/"} - declare_test! {GeneralStateTest_stRandom2, "GeneralStateTests/stRandom2/"} - declare_test! {GeneralStateTest_stRecursiveCreate, "GeneralStateTests/stRecursiveCreate/"} - declare_test! {GeneralStateTest_stRefundTest, "GeneralStateTests/stRefundTest/"} - declare_test! {GeneralStateTest_stReturnDataTest, "GeneralStateTests/stReturnDataTest/"} - declare_test! {GeneralStateTest_stRevertTest, "GeneralStateTests/stRevertTest/"} - declare_test! {GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"} - declare_test! {GeneralStateTest_stShift, "GeneralStateTests/stShift/"} - declare_test! {GeneralStateTest_stSolidityTest, "GeneralStateTests/stSolidityTest/"} - declare_test! {GeneralStateTest_stSpecialTest, "GeneralStateTests/stSpecialTest/"} - declare_test! {GeneralStateTest_stStackTests, "GeneralStateTests/stStackTests/"} - declare_test! {GeneralStateTest_stStaticCall, "GeneralStateTests/stStaticCall/"} - declare_test! {GeneralStateTest_stSystemOperationsTest, "GeneralStateTests/stSystemOperationsTest/"} - declare_test! {GeneralStateTest_stTransactionTest, "GeneralStateTests/stTransactionTest/"} - declare_test! {GeneralStateTest_stTransitionTest, "GeneralStateTests/stTransitionTest/"} - declare_test! {GeneralStateTest_stWalletTest, "GeneralStateTests/stWalletTest/"} - declare_test! {GeneralStateTest_stZeroCallsRevert, "GeneralStateTests/stZeroCallsRevert/"} - declare_test! {GeneralStateTest_stZeroCallsTest, "GeneralStateTests/stZeroCallsTest/"} - declare_test! {GeneralStateTest_stZeroKnowledge, "GeneralStateTests/stZeroKnowledge/"} - - // Attempts to send a transaction that requires more than current balance: - // Tx: - // https://github.com/ethereum/tests/blob/726b161ba8a739691006cc1ba080672bb50a9d49/GeneralStateTests/stZeroKnowledge2/ecmul_0-3_5616_28000_96.json#L170 - // Balance: - // https://github.com/ethereum/tests/blob/726b161ba8a739691006cc1ba080672bb50a9d49/GeneralStateTests/stZeroKnowledge2/ecmul_0-3_5616_28000_96.json#L126 - declare_test! {GeneralStateTest_stZeroKnowledge2, "GeneralStateTests/stZeroKnowledge2/"} -} diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index af066b288..bba800014 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -15,13 +15,8 @@ // along with Parity Ethereum. If not, see . pub use ethereum_types::{Address, H256, U256}; -use std::{ - collections::HashSet, - ffi::OsString, - fs::{read_dir, File}, - io::Read, - path::Path, -}; +use std::path::PathBuf; +use walkdir::{DirEntry, WalkDir}; /// Indicate when to run the hook passed to test functions. #[derive(Copy, Clone, Eq, PartialEq)] @@ -32,134 +27,11 @@ pub enum HookType { OnStop, } -pub fn run_test_path( - p: &Path, - skip: &[&'static str], - runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, - start_stop_hook: &mut H, -) { - let mut errors = Vec::new(); - run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors); - let empty: [String; 0] = []; - assert_eq!(errors, empty); -} - -fn run_test_path_inner( - p: &Path, - skip: &[&'static str], - runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, - start_stop_hook: &mut H, - errors: &mut Vec, -) { - let path = Path::new(p); - let s: HashSet = skip - .iter() - .map(|s| { - let mut os: OsString = s.into(); - os.push(".json"); - os - }) - .collect(); - let extension = path.extension().and_then(|s| s.to_str()); - if path.is_dir() { - for p in read_dir(path).unwrap().filter_map(|e| { - let e = e.unwrap(); - if s.contains(&e.file_name()) { - None - } else { - Some(e.path()) - } - }) { - run_test_path_inner(&p, skip, runner, start_stop_hook, errors); - } - } else if extension == Some("swp") || extension == None { - // Ignore junk - } else { - let mut path = p.to_path_buf(); - path.set_extension("json"); - run_test_file_append(&path, runner, start_stop_hook, errors) - } -} - -fn run_test_file_append( - path: &Path, - runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, - start_stop_hook: &mut H, - errors: &mut Vec, -) { - let mut data = Vec::new(); - let mut file = match File::open(&path) { - Ok(file) => file, - Err(_) => panic!("Error opening test file at: {:?}", path), - }; - file.read_to_end(&mut data) - .expect("Error reading test file"); - errors.append(&mut runner(&data, start_stop_hook)); -} - -pub fn run_test_file( - path: &Path, - runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, - start_stop_hook: &mut H, -) { - let mut data = Vec::new(); - let mut file = match File::open(&path) { - Ok(file) => file, - Err(_) => panic!("Error opening test file at: {:?}", path), - }; - file.read_to_end(&mut data) - .expect("Error reading test file"); - let results = runner(&data, start_stop_hook); - let empty: [String; 0] = []; - assert_eq!(results, empty); -} - -#[cfg(test)] -macro_rules! test { - ($name: expr, $skip: expr) => { - ::json_tests::test_common::run_test_path( - ::std::path::Path::new(concat!("res/ethereum/tests/", $name)), - &$skip, - do_json_test, - &mut |_, _| (), - ); - }; -} - -#[macro_export] -macro_rules! declare_test { - (skip => $arr: expr, $id: ident, $name: expr) => { - #[cfg(test)] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, $arr); - } - }; - (ignore => $id: ident, $name: expr) => { - #[cfg(test)] - #[ignore] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - }; - (heavy => $id: ident, $name: expr) => { - #[cfg(test)] - #[cfg(feature = "test-heavy")] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - }; - ($id: ident, $name: expr) => { - #[cfg(test)] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - }; +pub fn find_json_files_recursive(path: &PathBuf) -> Vec { + WalkDir::new(path) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with(".json")) + .map(DirEntry::into_path) + .collect::>() } diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index d7a0b6b05..6d8d503cd 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -22,37 +22,30 @@ use std::path::Path; use transaction_ext::Transaction; use types::{header::Header, transaction::UnverifiedTransaction}; -/// Run transaction jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) -} - -/// Run transaction jsontests on a given file. -pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, do_json_test, h) -} - -// Block number used to run the tests. -// Make sure that all the specified features are activated. -const BLOCK_NUMBER: u64 = 0x6ffffffffffffe; - -fn do_json_test( +pub fn json_transaction_test( + path: &Path, json_data: &[u8], start_stop_hook: &mut H, ) -> Vec { - let tests = ethjson::transaction::Test::load(json_data).unwrap(); + // Block number used to run the tests. + // Make sure that all the specified features are activated. + const BLOCK_NUMBER: u64 = 0x6ffffffffffffe; + + let tests = ethjson::transaction::Test::load(json_data).expect(&format!( + "Could not parse JSON transaction test data from {}", + path.display() + )); let mut failed = Vec::new(); for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); + println!(" - tx: {} ", name); + for (spec_name, result) in test.post_state { let spec = match EvmTestClient::spec_from_json(&spec_name) { Some(spec) => spec, None => { - println!( - " - {} | {:?} Ignoring tests because of missing spec", - name, spec_name - ); + failed.push(format!("{}-{:?} (missing spec)", name, spec_name)); continue; } }; @@ -110,14 +103,3 @@ fn do_json_test( } failed } - -declare_test! {TransactionTests_ttAddress, "TransactionTests/ttAddress"} -declare_test! {TransactionTests_ttData, "TransactionTests/ttData"} -declare_test! {TransactionTests_ttGasLimit, "TransactionTests/ttGasLimit"} -declare_test! {TransactionTests_ttGasPrice, "TransactionTests/ttGasPrice"} -declare_test! {TransactionTests_ttNonce, "TransactionTests/ttNonce"} -declare_test! {TransactionTests_ttRSValue, "TransactionTests/ttRSValue"} -declare_test! {TransactionTests_ttSignature, "TransactionTests/ttSignature"} -declare_test! {TransactionTests_ttValue, "TransactionTests/ttValue"} -declare_test! {TransactionTests_ttVValue, "TransactionTests/ttVValue"} -declare_test! {TransactionTests_ttWrongRLP, "TransactionTests/ttWrongRLP"} diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index 1745132e8..3048a773a 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -17,29 +17,29 @@ use ethereum_types::H256; use ethjson; use ethtrie::RlpCodec; +use std::path::Path; use trie::{TrieFactory, TrieSpec}; use super::HookType; -pub use self::{ - generic::{run_test_file as run_generic_test_file, run_test_path as run_generic_test_path}, - secure::{run_test_file as run_secure_test_file, run_test_path as run_secure_test_path}, -}; - -fn test_trie( +pub fn json_trie_test( + path: &Path, json: &[u8], trie: TrieSpec, start_stop_hook: &mut H, ) -> Vec { - let tests = ethjson::trie::Test::load(json).unwrap(); + let tests = ethjson::trie::Test::load(json).expect(&format!( + "Could not parse JSON trie test data from {}", + path.display() + )); let factory = TrieFactory::<_, RlpCodec>::new(trie); - let mut result = vec![]; + let mut failed = vec![]; for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); let mut memdb = journaldb::new_memory_db(); - let mut root = H256::default(); + let mut root = H256::zero(); let mut t = factory.create(&mut memdb, &mut root); for (key, value) in test.input.data.into_iter() { @@ -51,65 +51,14 @@ fn test_trie( )); } - if *t.root() != test.root.into() { - result.push(format!("Trie test '{:?}' failed.", name)); + if *t.root() == test.root.into() { + println!(" - trie: {}...OK", name); + } else { + println!(" - trie: {}...FAILED ({:?})", name, path); + failed.push(format!("{}", name)); } - start_stop_hook(&name, HookType::OnStop); } - for i in &result { - println!("FAILED: {}", i); - } - - result -} - -mod generic { - use std::path::Path; - use trie::TrieSpec; - - use super::HookType; - - /// Run generic trie jsontests on a given folder. - pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) - } - - /// Run generic trie jsontests on a given file. - pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, do_json_test, h) - } - - fn do_json_test(json: &[u8], h: &mut H) -> Vec { - super::test_trie(json, TrieSpec::Generic, h) - } - - declare_test! {TrieTests_trietest, "TrieTests/trietest"} - declare_test! {TrieTests_trieanyorder, "TrieTests/trieanyorder"} -} - -mod secure { - use std::path::Path; - use trie::TrieSpec; - - use super::HookType; - - /// Run secure trie jsontests on a given folder. - pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) - } - - /// Run secure trie jsontests on a given file. - pub fn run_test_file(p: &Path, h: &mut H) { - ::json_tests::test_common::run_test_file(p, do_json_test, h) - } - - fn do_json_test(json: &[u8], h: &mut H) -> Vec { - super::test_trie(json, TrieSpec::Secure, h) - } - - declare_test! {TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"} - declare_test! {TrieTests_trietest_secure, "TrieTests/trietest_secureTrie"} - declare_test! {TrieTests_trieanyorder_secure, "TrieTests/trieanyorder_secureTrie"} + failed } diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 8857751a3..3a51c6730 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -112,6 +112,8 @@ extern crate env_logger; extern crate ethcore_accounts as accounts; #[cfg(feature = "stratum")] extern crate ethcore_stratum; +#[cfg(feature = "json-tests")] +extern crate globset; #[cfg(any(test, feature = "test-helpers"))] extern crate kvdb_memorydb; #[cfg(any(test, feature = "kvdb-rocksdb"))] @@ -120,6 +122,10 @@ extern crate kvdb_rocksdb; extern crate rlp_compress; #[cfg(any(test, feature = "tempdir"))] extern crate tempdir; +#[cfg(feature = "json-tests")] +extern crate tempfile; +#[cfg(feature = "json-tests")] +extern crate walkdir; #[macro_use] extern crate ethabi_derive; diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index d03544d0a..2a1c6718d 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -148,10 +148,9 @@ fn run_stats_jsontests_vm(args: Args) { }); } }; - if !file.is_file() { - json_tests::run_executive_test_path(&file, &[], &mut record_time); - } else { - json_tests::run_executive_test_file(&file, &mut record_time); + for file_path in json_tests::find_json_files_recursive(&file) { + let json_data = std::fs::read(&file_path).unwrap(); + json_tests::json_executive_test(&file_path, &json_data, &mut record_time); } } diff --git a/json/src/blockchain/blockchain.rs b/json/src/blockchain/blockchain.rs index 8329a9321..549001850 100644 --- a/json/src/blockchain/blockchain.rs +++ b/json/src/blockchain/blockchain.rs @@ -49,7 +49,7 @@ pub struct BlockChain { /// Blocks. pub blocks: Vec, /// Post state. - pub post_state: State, + pub post_state: Option, /// Pre state. #[serde(rename = "pre")] pub pre_state: State, diff --git a/json/src/blockchain/state.rs b/json/src/blockchain/state.rs index 1431f012c..57211ef6d 100644 --- a/json/src/blockchain/state.rs +++ b/json/src/blockchain/state.rs @@ -16,19 +16,62 @@ //! Blockchain test state deserializer. -use blockchain::account::Account; -use hash::Address; +use crate::{ + bytes::Bytes, + hash::{Address, H256}, + spec::{Account, Builtin}, +}; +use serde::Deserialize; use std::collections::BTreeMap; -/// Blockchain test state deserializer. -#[derive(Debug, PartialEq, Deserialize, Clone)] -pub struct State(BTreeMap); +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum HashOrMap { + /// When the `postState` is large, tests sometimes just include the state root of the last + /// successful block here. + Hash(H256), + /// The expected `postState` of a test + Map(BTreeMap), +} + +/// Blockchain state deserializer. +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct State(pub HashOrMap); + +impl State { + /// Returns all builtins. + pub fn builtins(&self) -> BTreeMap { + match &self.0 { + HashOrMap::Hash(_) => BTreeMap::default(), + HashOrMap::Map(map) => map + .iter() + .filter_map(|(add, ref acc)| acc.builtin.clone().map(|b| (add.clone(), b.into()))) + .collect(), + } + } + + /// Returns all constructors. + pub fn constructors(&self) -> BTreeMap { + match &self.0 { + HashOrMap::Hash(_) => BTreeMap::default(), + HashOrMap::Map(map) => map + .iter() + .filter_map(|(add, ref acc)| acc.constructor.clone().map(|b| (add.clone(), b))) + .collect(), + } + } +} impl IntoIterator for State { type Item = as IntoIterator>::Item; type IntoIter = as IntoIterator>::IntoIter; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + if let HashOrMap::Map(m) = self.0 { + m.into_iter() + } else { + BTreeMap::default().into_iter() + } } } diff --git a/json/src/spec/account.rs b/json/src/spec/account.rs index c91bb8eec..e3462e39e 100644 --- a/json/src/spec/account.rs +++ b/json/src/spec/account.rs @@ -23,7 +23,7 @@ use spec::builtin::BuiltinCompat; use uint::Uint; /// Spec account. -#[derive(Debug, PartialEq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Account { diff --git a/json/src/spec/spec.rs b/json/src/spec/spec.rs index 3ad07353f..930b1081c 100644 --- a/json/src/spec/spec.rs +++ b/json/src/spec/spec.rs @@ -35,6 +35,7 @@ pub enum ForkSpec { FrontierToHomesteadAt5, HomesteadToDaoAt5, HomesteadToEIP150At5, + ByzantiumToConstantinopleAt5, } /// Spec deserialization. diff --git a/json/src/test/mod.rs b/json/src/test/mod.rs index 78430a0ba..cc393ee2a 100644 --- a/json/src/test/mod.rs +++ b/json/src/test/mod.rs @@ -18,7 +18,7 @@ use hash::H256; use serde_json::{self, Error}; -use std::{collections::BTreeMap, io::Read}; +use std::{collections::BTreeMap, io::Read, path::PathBuf}; use uint::Uint; /// Blockchain test header deserializer. @@ -119,3 +119,123 @@ impl SkipStates { } } } + +/// Describes a github.com/ethereum/tests suite +#[derive(Debug, PartialEq, Deserialize)] +pub struct EthereumTestSuite { + /// Blockchain tests + pub chain: Vec, + /// State tests + pub state: Vec, + /// Difficulty tests + pub difficulty: Vec, + /// Executive tests + pub executive: Vec, + /// Transaction tests + pub transaction: Vec, + /// Trie tests + pub trie: Vec, +} + +/// Chain spec used in tests +#[derive(Debug, PartialEq, Deserialize)] +pub enum TestChainSpec { + /// Foundation + Foundation, + /// ByzantiumTest + ByzantiumTest, + /// FrontierTest + FrontierTest, + /// HomesteadTest + HomesteadTest, +} + +/// Kind of trie used in test +#[derive(Debug, PartialEq, Deserialize)] +pub enum TestTrieSpec { + /// Generic + Generic, + /// Secure + Secure, +} + +/// A set of blockchain tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ChainTests { + /// Path of the json tests + pub path: PathBuf, + /// Tests to skip + pub skip: Vec, +} + +/// Tests to skip in chain tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ChainTestSkip { + /// Issue reference. + pub reference: String, + /// Test names to skip + pub names: Vec, + /// Test paths to skip + pub paths: Vec, +} + +/// A set of state tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateTests { + /// Path of the json tests + pub path: PathBuf, + /// Tests to skip + pub skip: Vec, +} + +/// State test to skip +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateTestSkip { + /// Issue reference. + pub reference: String, + /// Paths to skip + pub paths: Vec, + /// Test names to skip + pub names: BTreeMap, +} + +/// State subtest to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipSubStates1 { + /// State test number of this item. Or '*' for all state. + pub subnumbers: Vec, + /// Chain for this items. + pub chain: String, +} + +/// A set of difficulty tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct DifficultyTests { + /// Path of the json tests + pub path: Vec, + /// Chain spec to use + pub chainspec: TestChainSpec, +} + +/// A set of executive tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ExecutiveTests { + /// Path of the json tests + pub path: PathBuf, +} + +/// A set of transaction tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct TransactionTests { + /// Path of the json tests + pub path: PathBuf, +} + +/// A set of trie tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct TrieTests { + /// Path of the json tests + pub path: Vec, + /// Trie spec to use + pub triespec: TestTrieSpec, +}