From e9f1b389840bf3c963b9b296f2df9fcd5c229921 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 17:21:45 +0800 Subject: [PATCH] EVM benchmark utilities (#8944) * Make it possible to expose jsontests using feature flag * Add start_stop_hook for jsontests * Fix evmbin compile * Implement vm jsontests times to tsv result * Use /usr/bin/env to avoid errors on non-Debian systems * Move evmbin/bench.sh to scripts and add vm_jsontests script for convenience * Add tempdir as required deps for test-helpers * Address grumbles on comments * Detect file/folder automatically and add command docs * Fix bench script * times -> timings * typo: wrong if condition --- ethcore/Cargo.toml | 6 +-- ethcore/src/client/evm_test_client.rs | 23 +++------- ethcore/src/json_tests/chain.rs | 27 +++++++++-- ethcore/src/json_tests/difficulty.rs | 18 +++++--- ethcore/src/json_tests/executive.rs | 27 ++++++++--- ethcore/src/json_tests/mod.rs | 19 ++++++++ ethcore/src/json_tests/state.rs | 26 +++++++++-- ethcore/src/json_tests/test_common.rs | 34 +++++++++++--- ethcore/src/json_tests/transaction.rs | 17 ++++++- ethcore/src/json_tests/trie.rs | 47 +++++++++++++++++--- ethcore/src/lib.rs | 4 +- evmbin/Cargo.toml | 2 +- evmbin/src/info.rs | 2 +- evmbin/src/main.rs | 46 ++++++++++++++++++- scripts/aura-test.sh | 2 +- scripts/cov.sh | 2 +- scripts/doc.sh | 2 +- scripts/docker-build.sh | 2 +- scripts/evm_jsontests_bench.sh | 16 +++++++ evmbin/bench.sh => scripts/evm_uint_bench.sh | 26 ++++++----- scripts/gitlab-build.sh | 2 +- scripts/gitlab-push-release.sh | 2 +- scripts/gitlab-test.sh | 2 +- scripts/hook.sh | 2 +- scripts/safe_curl.sh | 3 +- scripts/validate_chainspecs.sh | 3 +- 26 files changed, 284 insertions(+), 78 deletions(-) create mode 100755 scripts/evm_jsontests_bench.sh rename evmbin/bench.sh => scripts/evm_uint_bench.sh (54%) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index fe05badb4..3c99e01aa 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -67,7 +67,7 @@ keccak-hash = { path = "../util/hash" } triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } -tempdir = "0.3" +tempdir = { version = "0.3", optional = true } kvdb-rocksdb = { path = "../util/kvdb-rocksdb" } [dev-dependencies] @@ -84,10 +84,10 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] # EVM debug traces are printed. slow-blocks = [] # Run JSON consensus tests. -json-tests = ["ethcore-transaction/json-tests"] +json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"] # Run memory/cpu heavy tests. test-heavy = [] # Compile benches benches = [] # Compile test helpers -test-helpers = [] +test-helpers = ["tempdir"] diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index fbf57cbdc..2e5e912cc 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -64,15 +64,6 @@ impl fmt::Display for EvmTestError { use ethereum; use ethjson::state::test::ForkSpec; -lazy_static! { - pub static ref FRONTIER: spec::Spec = ethereum::new_frontier_test(); - pub static ref HOMESTEAD: spec::Spec = ethereum::new_homestead_test(); - pub static ref EIP150: spec::Spec = ethereum::new_eip150_test(); - pub static ref EIP161: spec::Spec = ethereum::new_eip161_test(); - pub static ref BYZANTIUM: spec::Spec = ethereum::new_byzantium_test(); - pub static ref BYZANTIUM_TRANSITION: spec::Spec = ethereum::new_transition_test(); -} - /// Simplified, single-block EVM test client. pub struct EvmTestClient<'a> { state: state::State, @@ -90,14 +81,14 @@ impl<'a> fmt::Debug for EvmTestClient<'a> { impl<'a> EvmTestClient<'a> { /// Converts a json spec definition into spec. - pub fn spec_from_json(spec: &ForkSpec) -> Option<&'static spec::Spec> { + pub fn spec_from_json(spec: &ForkSpec) -> Option { match *spec { - ForkSpec::Frontier => Some(&*FRONTIER), - ForkSpec::Homestead => Some(&*HOMESTEAD), - ForkSpec::EIP150 => Some(&*EIP150), - ForkSpec::EIP158 => Some(&*EIP161), - ForkSpec::Byzantium => Some(&*BYZANTIUM), - ForkSpec::EIP158ToByzantiumAt5 => Some(&BYZANTIUM_TRANSITION), + ForkSpec::Frontier => Some(ethereum::new_frontier_test()), + ForkSpec::Homestead => Some(ethereum::new_homestead_test()), + ForkSpec::EIP150 => Some(ethereum::new_eip150_test()), + ForkSpec::EIP158 => Some(ethereum::new_eip161_test()), + ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()), + ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()), ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, _ => None, } diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 642225e30..2d643e75f 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use std::sync::Arc; use client::{EvmTestClient, Client, ClientConfig, ChainInfo, ImportBlock}; use block::Block; @@ -23,12 +24,26 @@ use miner::Miner; use io::IoChannel; use test_helpers; -pub fn json_chain_test(json_data: &[u8]) -> Vec { +use super::HookType; + +/// Run chain jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) +} + +/// Run chain jsontests on a given file. +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, json_chain_test, h) +} + +pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::blockchain::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut fail = false; { let mut fail_unless = |cond: bool| if !cond && !fail { @@ -42,7 +57,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { let spec = { let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) { - Some(spec) => (*spec).clone(), + Some(spec) => spec, None => { println!(" - {} | {:?} Ignoring tests because of missing spec", name, blockchain.network); continue; @@ -82,17 +97,21 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { if !fail { flushln!("ok"); } + + start_stop_hook(&name, HookType::OnStop); } println!("!!! {:?} tests from failed.", failed.len()); failed } +#[cfg(test)] mod block_tests { use super::json_chain_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_chain_test(json_data) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_chain_test(json_data, h) } declare_test!{BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} diff --git a/ethcore/src/json_tests/difficulty.rs b/ethcore/src/json_tests/difficulty.rs index d111f0890..2647e1659 100644 --- a/ethcore/src/json_tests/difficulty.rs +++ b/ethcore/src/json_tests/difficulty.rs @@ -19,12 +19,16 @@ use header::Header; use ethereum_types::U256; use spec::Spec; -pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec { +use super::HookType; + +pub fn json_difficulty_test(json_data: &[u8], spec: Spec, start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::test::DifficultyTest::load(json_data).unwrap(); let engine = &spec.engine; for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + flush!(" - {}...", name); println!(" - {}...", name); @@ -42,15 +46,18 @@ pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec { let expected_difficulty: U256 = test.current_difficulty.into(); assert_eq!(header.difficulty(), &expected_difficulty); flushln!("ok"); + + start_stop_hook(&name, HookType::OnStop); } vec![] } mod difficulty_test_byzantium { use super::json_difficulty_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_difficulty_test(json_data, ::ethereum::new_byzantium_test()) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_difficulty_test(json_data, ::ethereum::new_byzantium_test(), h) } declare_test!{DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"} @@ -59,10 +66,11 @@ mod difficulty_test_byzantium { mod difficulty_test_foundation { use super::json_difficulty_test; use tempdir::TempDir; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { let tempdir = TempDir::new("").unwrap(); - json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path())) + json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()), h) } declare_test!{DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"} diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 5bda6c55a..86c6d6efc 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use std::sync::Arc; use super::test_common::*; use state::{Backend as StateBackend, State, Substate}; @@ -35,6 +36,18 @@ use rlp::RlpStream; use hash::keccak; use machine::EthereumMachine as Machine; +use super::HookType; + +/// Run executive jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) +} + +/// Run executive jsontests on a given file. +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) +} + #[derive(Debug, PartialEq, Clone)] struct CallCreate { data: Bytes, @@ -193,20 +206,22 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B> } } -fn do_json_test(json_data: &[u8]) -> Vec { +fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { let vms = VMType::all(); vms .iter() - .flat_map(|vm| do_json_test_for(vm, json_data)) + .flat_map(|vm| do_json_test_for(vm, json_data, h)) .collect() } -fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec { +fn do_json_test_for(vm_type: &VMType, json_data: &[u8], start_stop_hook: &mut H) -> Vec { let tests = ethjson::vm::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, vm) in tests.into_iter() { - println!("name: {:?}", name); + start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart); + + info!(target: "jsontests", "name: {:?}", name); let mut fail = false; let mut fail_unless = |cond: bool, s: &str | if !cond && !fail { @@ -305,10 +320,12 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec { fail_unless(Some(callcreates) == calls, "callcreates does not match"); } }; + + start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop); } for f in &failed { - println!("FAILED: {:?}", f); + error!("FAILED: {:?}", f); } failed diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index 65cc6d213..fa1c822ce 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +//! Helpers and tests for operating on jsontests. + #[macro_use] mod test_common; @@ -22,4 +24,21 @@ mod executive; mod state; mod chain; mod trie; + +#[cfg(test)] mod difficulty; + +pub use self::test_common::HookType; + +pub use self::transaction::run_test_path as run_transaction_test_path; +pub use self::transaction::run_test_file as run_transaction_test_file; +pub use self::executive::run_test_path as run_executive_test_path; +pub use self::executive::run_test_file as run_executive_test_file; +pub use self::state::run_test_path as run_state_test_path; +pub use self::state::run_test_file as run_state_test_file; +pub use self::chain::run_test_path as run_chain_test_path; +pub use self::chain::run_test_file as run_chain_test_file; +pub use self::trie::run_generic_test_path as run_generic_trie_test_path; +pub use self::trie::run_generic_test_file as run_generic_trie_test_file; +pub use self::trie::run_secure_test_path as run_secure_trie_test_path; +pub use self::trie::run_secure_test_file as run_secure_trie_test_file; diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 45ec6f3fb..da1716972 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use super::test_common::*; use pod_state::PodState; use trace; @@ -22,12 +23,26 @@ use ethjson; use transaction::SignedTransaction; use vm::EnvInfo; -pub fn json_chain_test(json_data: &[u8]) -> Vec { +use super::HookType; + +/// Run state jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) +} + +/// Run state jsontests on a given file. +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, json_chain_test, h) +} + +pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + { let multitransaction = test.transaction; let env: EnvInfo = test.env.into(); @@ -50,7 +65,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { let transaction: SignedTransaction = multitransaction.select(&state.indexes).into(); let result = || -> Result<_, EvmTestError> { - Ok(EvmTestClient::from_pod_state(spec, pre.clone())? + Ok(EvmTestClient::from_pod_state(&spec, pre.clone())? .transact(&env, transaction, trace::NoopTracer, trace::NoopVMTracer)) }; match result() { @@ -81,6 +96,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { } } + start_stop_hook(&name, HookType::OnStop); } if !failed.is_empty() { @@ -89,11 +105,13 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { failed } +#[cfg(test)] mod state_tests { use super::json_chain_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_chain_test(json_data) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_chain_test(json_data, h) } declare_test!{GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"} diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index 6ce38b27a..39b7447c7 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -21,7 +21,20 @@ use std::path::Path; use std::ffi::OsString; pub use ethereum_types::{H256, U256, Address}; -pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u8]) -> Vec) { +/// Indicate when to run the hook passed to test functions. +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum HookType { + /// Hook to code to run on test start. + OnStart, + /// Hook to code to run on test end. + OnStop +} + +pub fn run_test_path( + p: &Path, skip: &[&'static str], + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H +) { let path = Path::new(p); let s: HashSet = skip.iter().map(|s| { let mut os: OsString = s.into(); @@ -36,33 +49,39 @@ pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u } else { Some(e.path()) }}) { - run_test_path(&p, skip, runner) + run_test_path(&p, skip, runner, start_stop_hook) } } else { let mut path = p.to_path_buf(); path.set_extension("json"); - run_test_file(&path, runner) + run_test_file(&path, runner, start_stop_hook) } } -pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec) { +pub fn run_test_file( + path: &Path, + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H +) { let mut data = Vec::new(); let mut file = File::open(&path).expect("Error opening test file"); file.read_to_end(&mut data).expect("Error reading test file"); - let results = runner(&data); + let results = runner(&data, start_stop_hook); let empty: [String; 0] = []; assert_eq!(results, empty); } +#[cfg(test)] macro_rules! test { ($name: expr, $skip: expr) => { - ::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test); + ::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test, &mut |_, _| ()); } } #[macro_export] macro_rules! declare_test { (skip => $arr: expr, $id: ident, $name: expr) => { + #[cfg(test)] #[test] #[allow(non_snake_case)] fn $id() { @@ -70,6 +89,7 @@ macro_rules! declare_test { } }; (ignore => $id: ident, $name: expr) => { + #[cfg(test)] #[ignore] #[test] #[allow(non_snake_case)] @@ -78,6 +98,7 @@ macro_rules! declare_test { } }; (heavy => $id: ident, $name: expr) => { + #[cfg(test)] #[cfg(feature = "test-heavy")] #[test] #[allow(non_snake_case)] @@ -86,6 +107,7 @@ macro_rules! declare_test { } }; ($id: ident, $name: expr) => { + #[cfg(test)] #[test] #[allow(non_snake_case)] fn $id() { diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index 295093305..af16481f4 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -14,19 +14,32 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use super::test_common::*; use evm; use ethjson; use rlp::Rlp; use transaction::{Action, UnverifiedTransaction, SignedTransaction}; -fn do_json_test(json_data: &[u8]) -> Vec { +/// Run transaction jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) +} + +/// Run transaction jsontests on a given file. +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) +} + +fn do_json_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { let tests = ethjson::transaction::Test::load(json_data).unwrap(); let mut failed = Vec::new(); let frontier_schedule = evm::Schedule::new_frontier(); let homestead_schedule = evm::Schedule::new_homestead(); let byzantium_schedule = evm::Schedule::new_byzantium(); for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut fail_unless = |cond: bool, title: &str| if !cond { failed.push(name.clone()); println!("Transaction failed: {:?}: {:?}", name, title); }; let number: Option = test.block_number.map(Into::into); @@ -69,6 +82,8 @@ fn do_json_test(json_data: &[u8]) -> Vec { Action::Create => fail_unless(None == to, "create mismatch"), } } + + start_stop_hook(&name, HookType::OnStop); } for f in &failed { diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index fae7cc738..81bc5bcc0 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -19,12 +19,21 @@ use trie::{TrieFactory, TrieSpec}; use ethereum_types::H256; use memorydb::MemoryDB; -fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { +use super::HookType; + +pub use self::generic::run_test_path as run_generic_test_path; +pub use self::generic::run_test_file as run_generic_test_file; +pub use self::secure::run_test_path as run_secure_test_path; +pub use self::secure::run_test_file as run_secure_test_file; + +fn test_trie(json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); let factory = TrieFactory::new(trie); let mut result = vec![]; for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut memdb = MemoryDB::new(); let mut root = H256::default(); let mut t = factory.create(&mut memdb, &mut root); @@ -39,6 +48,8 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { if *t.root() != test.root.into() { result.push(format!("Trie test '{:?}' failed.", name)); } + + start_stop_hook(&name, HookType::OnStop); } for i in &result { @@ -49,10 +60,23 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { } mod generic { + use std::path::Path; use trie::TrieSpec; - fn do_json_test(json: &[u8]) -> Vec { - super::test_trie(json, TrieSpec::Generic) + use super::HookType; + + /// Run generic trie jsontests on a given folder. + pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) + } + + /// Run generic trie jsontests on a given file. + pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) + } + + fn do_json_test(json: &[u8], h: &mut H) -> Vec { + super::test_trie(json, TrieSpec::Generic, h) } declare_test!{TrieTests_trietest, "TrieTests/trietest"} @@ -60,10 +84,23 @@ mod generic { } mod secure { + use std::path::Path; use trie::TrieSpec; - fn do_json_test(json: &[u8]) -> Vec { - super::test_trie(json, TrieSpec::Secure) + use super::HookType; + + /// Run secure trie jsontests on a given folder. + pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) + } + + /// Run secure trie jsontests on a given file. + pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) + } + + fn do_json_test(json: &[u8], h: &mut H) -> Vec { + super::test_trie(json, TrieSpec::Secure, h) } declare_test!{TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 274d52b24..8d8b22c6a 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -109,6 +109,7 @@ extern crate vm; extern crate wasm; extern crate memory_cache; extern crate journaldb; +#[cfg(any(test, feature = "json-tests", feature = "test-helpers"))] extern crate tempdir; #[macro_use] @@ -168,9 +169,8 @@ mod tx_filter; #[cfg(test)] mod tests; -#[cfg(test)] #[cfg(feature = "json-tests")] -mod json_tests; +pub mod json_tests; #[cfg(any(test, feature = "test-helpers"))] pub mod test_helpers; diff --git a/evmbin/Cargo.toml b/evmbin/Cargo.toml index 51865091a..9ff27a2e6 100644 --- a/evmbin/Cargo.toml +++ b/evmbin/Cargo.toml @@ -10,7 +10,7 @@ path = "./src/main.rs" [dependencies] docopt = "0.8" -ethcore = { path = "../ethcore", features = ["test-helpers"] } +ethcore = { path = "../ethcore", features = ["test-helpers", "json-tests"] } ethjson = { path = "../json" } ethcore-bytes = { path = "../util/bytes" } ethcore-transaction = { path = "../ethcore/transaction" } diff --git a/evmbin/src/info.rs b/evmbin/src/info.rs index d1cd3cf6f..ce824fe17 100644 --- a/evmbin/src/info.rs +++ b/evmbin/src/info.rs @@ -105,7 +105,7 @@ pub fn run_transaction( informant.set_gas(env_info.gas_limit); - let result = run(spec, env_info.gas_limit, pre_state, |mut client| { + let result = run(&spec, env_info.gas_limit, pre_state, |mut client| { let result = client.transact(env_info, transaction, trace::NoopTracer, informant); match result { TransactResult::Ok { state_root, .. } if state_root != post_root => { diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index 4620143f7..fb0c35b07 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -46,7 +46,7 @@ use docopt::Docopt; use rustc_hex::FromHex; use ethereum_types::{U256, Address}; use bytes::Bytes; -use ethcore::spec; +use ethcore::{spec, json_tests}; use vm::{ActionParams, CallType}; mod info; @@ -61,9 +61,16 @@ EVM implementation for Parity. Usage: parity-evm state-test [--json --std-json --only NAME --chain CHAIN] parity-evm stats [options] + parity-evm stats-jsontests-vm parity-evm [options] parity-evm [-h | --help] +Commands: + state-test Run a state test from a json file. + stats Execute EVM runtime code and return the statistics. + stats-jsontests-vm Execute standard jsontests format VMTests and return + timing statistcis in tsv format. + Transaction options: --code CODE Contract code as hex (without 0x). --to ADDRESS Recipient address (without 0x). @@ -90,6 +97,8 @@ fn main() { if args.cmd_state_test { run_state_test(args) + } else if args.cmd_stats_jsontests_vm { + run_stats_jsontests_vm(args) } else if args.flag_json { run_call(args, display::json::Informant::default()) } else if args.flag_std_json { @@ -99,6 +108,40 @@ fn main() { } } +fn run_stats_jsontests_vm(args: Args) { + use json_tests::HookType; + use std::collections::HashMap; + use std::time::{Instant, Duration}; + + let file = args.arg_file.expect("FILE (or PATH) is required"); + + let mut timings: HashMap)> = HashMap::new(); + + { + let mut record_time = |name: &str, typ: HookType| { + match typ { + HookType::OnStart => { + timings.insert(name.to_string(), (Instant::now(), None)); + }, + HookType::OnStop => { + timings.entry(name.to_string()).and_modify(|v| { + v.1 = Some(v.0.elapsed()); + }); + }, + } + }; + if !file.is_file() { + json_tests::run_executive_test_path(&file, &[], &mut record_time); + } else { + json_tests::run_executive_test_file(&file, &mut record_time); + } + } + + for (name, v) in timings { + println!("{}\t{}", name, display::as_micros(&v.1.expect("All hooks are called with OnStop; qed"))); + } +} + fn run_state_test(args: Args) { use ethjson::state::test::Test; @@ -179,6 +222,7 @@ fn run_call(args: Args, informant: T) { struct Args { cmd_stats: bool, cmd_state_test: bool, + cmd_stats_jsontests_vm: bool, arg_file: Option, flag_only: Option, flag_from: Option, diff --git a/scripts/aura-test.sh b/scripts/aura-test.sh index 1cd6bf536..0c39e12f4 100755 --- a/scripts/aura-test.sh +++ b/scripts/aura-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/cov.sh b/scripts/cov.sh index b199da31a..578fc715c 100755 --- a/scripts/cov.sh +++ b/scripts/cov.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Installing KCOV under ubuntu # https://users.rust-lang.org/t/tutorial-how-to-collect-test-coverages-for-rust-project/650# ### Install deps diff --git a/scripts/doc.sh b/scripts/doc.sh index f0022610a..44e544c99 100755 --- a/scripts/doc.sh +++ b/scripts/doc.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # generate documentation only for partiy and ethcore libraries cargo doc --no-deps --verbose --all --exclude parity-ipfs-api && diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 1adb49c1d..b880d33b7 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash cd docker/hub DOCKER_BUILD_TAG=$1 echo "Docker build tag: " $DOCKER_BUILD_TAG diff --git a/scripts/evm_jsontests_bench.sh b/scripts/evm_jsontests_bench.sh new file mode 100755 index 000000000..acec90219 --- /dev/null +++ b/scripts/evm_jsontests_bench.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +cargo build --release -p evmbin + +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmArithmeticTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBitwiseLogicOperation +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBlockInfoTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmEnvironmentalInfo +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmIOandFlowOperations +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmLogTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPerformance +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPushDupSwapTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmRandomTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSha3Test +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSystemOperations +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmTests diff --git a/evmbin/bench.sh b/scripts/evm_uint_bench.sh similarity index 54% rename from evmbin/bench.sh rename to scripts/evm_uint_bench.sh index 9d19729a6..b0ca1c4f7 100755 --- a/evmbin/bench.sh +++ b/scripts/evm_uint_bench.sh @@ -1,23 +1,25 @@ -#!/bin/bash +#!/usr/bin/env bash -set -e - -cargo build --release +cargo build --release -p evmbin # LOOP TEST CODE1=606060405260005b620f42408112156019575b6001016007565b600081905550600680602b6000396000f3606060405200 -ethvm --code $CODE1 -echo "^^^^ ethvm" -./target/release/evm stats --code $CODE1 --gas 4402000 +if [ -x "$(command -v ethvm)" ]; then + ethvm --code $CODE1 + echo "^^^^ ethvm" +fi +./target/release/parity-evm stats --code $CODE1 --gas 4402000 echo "^^^^ usize" -./target/release/evm stats --code $CODE1 +./target/release/parity-evm stats --code $CODE1 echo "^^^^ U256" # RNG TEST CODE2=6060604052600360056007600b60005b620f4240811215607f5767ffe7649d5eca84179490940267f47ed85c4b9a6379019367f8e5dd9a5c994bba9390930267f91d87e4b8b74e55019267ff97f6f3b29cda529290920267f393ada8dd75c938019167fe8d437c45bb3735830267f47d9a7b5428ffec019150600101600f565b838518831882186000555050505050600680609a6000396000f3606060405200 -ethvm --code $CODE2 -echo "^^^^ ethvm" -./target/release/evm stats --code $CODE2 --gas 143020115 +if [ -x "$(command -v ethvm)" ]; then + ethvm --code $CODE2 + echo "^^^^ ethvm" +fi +./target/release/parity-evm stats --code $CODE2 --gas 143020115 echo "^^^^ usize" -./target/release/evm stats --code $CODE2 +./target/release/parity-evm stats --code $CODE2 echo "^^^^ U256" diff --git a/scripts/gitlab-build.sh b/scripts/gitlab-build.sh index 5d1408e74..3d7d392d8 100755 --- a/scripts/gitlab-build.sh +++ b/scripts/gitlab-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/gitlab-push-release.sh b/scripts/gitlab-push-release.sh index 275576b91..207d66c11 100755 --- a/scripts/gitlab-push-release.sh +++ b/scripts/gitlab-push-release.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/gitlab-test.sh b/scripts/gitlab-test.sh index 57cf7c6b6..fbd93167a 100755 --- a/scripts/gitlab-test.sh +++ b/scripts/gitlab-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #ARGUMENT test for RUST and COVERAGE set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/hook.sh b/scripts/hook.sh index 2d64d5782..ed7e173c5 100755 --- a/scripts/hook.sh +++ b/scripts/hook.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh FILE=./.git/hooks/pre-push echo "#!/bin/sh\n" > $FILE diff --git a/scripts/safe_curl.sh b/scripts/safe_curl.sh index 5990d1e24..f5bb2ee4e 100755 --- a/scripts/safe_curl.sh +++ b/scripts/safe_curl.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu @@ -17,4 +17,3 @@ else echo 'Unable to push info to updater service.'; exit 2 fi - diff --git a/scripts/validate_chainspecs.sh b/scripts/validate_chainspecs.sh index a0e2d9b17..c350445dd 100755 --- a/scripts/validate_chainspecs.sh +++ b/scripts/validate_chainspecs.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh ERR=0 cargo build --release -p chainspec @@ -12,4 +12,3 @@ for spec in ethcore/res/ethereum/*.json; do done exit $ERR -