Implement JSON test suite (#11801)

This commit is contained in:
adria0.eth 2020-09-08 02:48:09 +02:00 committed by Artem Vorotnikov
parent 506cee52e8
commit c84d82580a
No known key found for this signature in database
GPG Key ID: E0148C3F2FBB7A20
20 changed files with 777 additions and 600 deletions

18
Cargo.lock generated
View File

@ -830,6 +830,7 @@ dependencies = [
"ethkey 0.3.0",
"evm 0.1.0",
"fetch 0.1.0",
"globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)",
"hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -861,8 +862,10 @@ dependencies = [
"rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)",
"stats 0.1.0",
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"time-utils 0.1.0",
"trace-time 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -871,6 +874,7 @@ dependencies = [
"unexpected 0.1.0",
"using_queue 0.1.0",
"vm 0.1.0",
"walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
"wasm 0.1.0",
]
@ -3865,6 +3869,19 @@ dependencies = [
"remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempfile"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "term_size"
version = "0.3.1"
@ -4984,6 +5001,7 @@ dependencies = [
"checksum synstructure 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "575be94ccb86e8da37efb894a87e2b660be299b41d8ef347f9d6d79fbe61b1ba"
"checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
"checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327"
"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"

View File

@ -31,6 +31,7 @@ ethereum-types = "0.4"
ethjson = { path = "../json" }
ethkey = { path = "../accounts/ethkey" }
evm = { path = "evm" }
globset = "0.4"
hash-db = "0.11.0"
heapsize = "0.4"
itertools = "0.5"
@ -60,14 +61,17 @@ rlp_derive = { path = "../util/rlp-derive" }
rustc-hex = "1.0"
serde = "1.0"
serde_derive = "1.0"
serde_json = "1.0"
stats = { path = "../util/stats" }
tempdir = {version="0.3", optional = true}
tempfile = "3.1.0"
time-utils = { path = "../util/time-utils" }
trace-time = "0.1"
triehash-ethereum = { version = "0.2", path = "../util/triehash-ethereum" }
unexpected = { path = "../util/unexpected" }
using_queue = { path = "../miner/using-queue" }
vm = { path = "vm" }
walkdir = "2"
wasm = { path = "wasm" }
[dev-dependencies]

View File

@ -0,0 +1,51 @@
{
"chain": [
{
"path": "res/ethereum/tests/BlockchainTests",
"skip" : []
}
],
"state": [
{
"path": "res/ethereum/tests/GeneralStateTests",
"skip" : []
}
],
"difficulty": [
{
"path": [
"res/ethereum/tests/BasicTests/difficulty.json",
"res/ethereum/tests/BasicTests/difficultyMainNetwork.json"
],
"chainspec": "Foundation"
}
],
"executive": [
{
"path": "res/ethereum/tests/VMTests"
}
],
"transaction": [
{
"path": "res/ethereum/tests/TransactionTests"
}
],
"trie": [
{
"path": [
"res/ethereum/tests/TrieTests/trietest.json",
"res/ethereum/tests/TrieTests/trieanyorder.json"
],
"triespec": "Generic"
},
{
"path": [
"res/ethereum/tests/TrieTests/hex_encoded_securetrie_test.json",
"res/ethereum/tests/TrieTests/trietest_secureTrie.json",
"res/ethereum/tests/TrieTests/trieanyorder_secureTrie.json"
],
"triespec": "Secure"
}
]
}

View File

@ -107,6 +107,7 @@ impl<'a> EvmTestClient<'a> {
ForkSpec::FrontierToHomesteadAt5
| ForkSpec::HomesteadToDaoAt5
| ForkSpec::HomesteadToEIP150At5 => None,
ForkSpec::ByzantiumToConstantinopleAt5 => None,
}
}

View File

@ -14,50 +14,135 @@
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
use super::{HookType, SKIP_TEST_STATE};
use client::{ChainInfo, Client, ClientConfig, EvmTestClient, ImportBlock};
use super::HookType;
use client::{
Balance, BlockChainClient, BlockId, ChainInfo, Client, ClientConfig, EvmTestClient,
ImportBlock, Nonce, StateOrBlock,
};
use ethereum_types::{H256, U256};
use ethjson;
use io::IoChannel;
use log::warn;
use miner::Miner;
use rustc_hex::ToHex;
use spec::Genesis;
use std::{path::Path, sync::Arc};
use test_helpers;
use verification::{queue::kind::blocks::Unverified, VerifierType};
/// Run chain jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
fn check_poststate(
client: &Arc<Client>,
test_name: &str,
post_state: ethjson::blockchain::State,
) -> bool {
let mut success = true;
for (address, expected) in post_state {
if let Some(expected_balance) = expected.balance {
let expected_balance: U256 = expected_balance.into();
let current_balance = client
.balance(
&address.clone().into(),
StateOrBlock::Block(BlockId::Latest),
)
.unwrap();
if expected_balance != current_balance {
warn!(target: "json-tests", "{} Poststate {:?} balance mismatch current={} expected={}",
test_name, address, current_balance, expected_balance);
success = false;
}
}
/// Run chain jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, json_chain_test, h)
if let Some(expected_nonce) = expected.nonce {
let expected_nonce: U256 = expected_nonce.into();
let current_nonce = client
.nonce(&address.clone().into(), BlockId::Latest)
.unwrap();
if expected_nonce != current_nonce {
warn!(target: "json-tests", "{} Poststate {:?} nonce mismatch current={} expected={}",
test_name, address, current_nonce, expected_nonce);
success = false;
}
}
fn skip_test(name: &String) -> bool {
SKIP_TEST_STATE
.block
.iter()
.any(|block_test| block_test.subtests.contains(name))
if let Some(expected_code) = expected.code {
let expected_code: String = expected_code.to_hex();
let current_code = match client.code(
&address.clone().into(),
StateOrBlock::Block(BlockId::Latest),
) {
Some(Some(code)) => code.to_hex(),
_ => "".to_string(),
};
if current_code != expected_code {
warn!(target: "json-tests", "{} Poststate {:?} code mismatch current={} expected={}",
test_name, address, current_code, expected_code);
success = false;
}
}
if let Some(expected_storage) = expected.storage {
for (uint_position, uint_expected_value) in expected_storage.iter() {
let mut position = H256::default();
uint_position.0.to_big_endian(position.as_mut());
let mut expected_value = H256::default();
uint_expected_value.0.to_big_endian(expected_value.as_mut());
let current_value = client
.storage_at(
&address.clone().into(),
&position,
StateOrBlock::Block(BlockId::Latest),
)
.unwrap();
if current_value != expected_value {
let position: &[u8] = position.as_ref();
let current_value: &[u8] = current_value.as_ref();
let expected_value: &[u8] = expected_value.as_ref();
warn!(target: "json-tests", "{} Poststate {:?} state {} mismatch actual={} expected={}",
test_name, address, position.to_hex(), current_value.to_hex(),
expected_value.to_hex());
success = false;
}
}
}
if expected.builtin.is_some() {
warn!(target: "json-tests", "{} Poststate {:?} builtin not supported", test_name, address);
success = false;
}
if expected.constructor.is_some() {
warn!(target: "json-tests", "{} Poststate {:?} constructor not supported", test_name, address);
success = false;
}
}
success
}
pub fn json_chain_test<H: FnMut(&str, HookType)>(
test: &ethjson::test::ChainTests,
path: &Path,
json_data: &[u8],
start_stop_hook: &mut H,
) -> Vec<String> {
let _ = ::env_logger::try_init();
let tests = ethjson::blockchain::Test::load(json_data).unwrap();
let tests = ethjson::blockchain::Test::load(json_data).expect(&format!(
"Could not parse JSON chain test data from {}",
path.display()
));
let mut failed = Vec::new();
for (name, blockchain) in tests.into_iter() {
if skip_test(&name) {
println!(
" - {} | {:?} Ignoring tests because in skip list",
name, blockchain.network
);
let skip_test = test
.skip
.iter()
.any(|block_test| block_test.names.contains(&name));
if skip_test {
info!(" SKIPPED {:?} {:?}", name, blockchain.network);
continue;
}
start_stop_hook(&name, HookType::OnStart);
let mut fail = false;
{
@ -72,14 +157,12 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
}
};
flush!(" - {}...", name);
let spec = {
let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) {
Some(spec) => spec,
None => {
println!(
" - {} | {:?} Ignoring tests because of missing spec",
info!(
" SKIPPED {:?} {:?} - Unimplemented chainspec ",
name, blockchain.network
);
continue;
@ -91,10 +174,11 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
spec.set_genesis_state(state)
.expect("Failed to overwrite genesis state");
spec.overwrite_genesis_params(genesis);
assert!(spec.is_state_root_valid());
spec
};
start_stop_hook(&name, HookType::OnStart);
{
let db = test_helpers::new_db();
let mut config = ClientConfig::default();
@ -103,6 +187,7 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
config.check_seal = false;
}
config.history = 8;
config.queue.verifier_settings.num_verifiers = 1;
let client = Client::new(
config,
&spec,
@ -110,103 +195,49 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
Arc::new(Miner::new_for_tests(&spec, None)),
IoChannel::disconnected(),
)
.unwrap();
.expect("Failed to instantiate a new Client");
for b in blockchain.blocks_rlp() {
if let Ok(block) = Unverified::from_rlp(b) {
let _ = client.import_block(block);
let bytes_len = b.len();
let block = Unverified::from_rlp(b);
match block {
Ok(block) => {
let num = block.header.number();
trace!(target: "json-tests", "{} Importing {} bytes. Block #{}", name, bytes_len, num);
let res = client.import_block(block);
if let Err(e) = res {
warn!(target: "json-tests", "{} Error importing block #{}: {:?}", name, num, e);
}
client.flush_queue();
client.import_verified_blocks();
}
Err(decoder_err) => {
warn!(target: "json-tests", "Error decoding test block: {:?} ({} bytes)", decoder_err, bytes_len);
}
fail_unless(client.chain_info().best_block_hash == blockchain.best_block.into());
}
}
if !fail {
flushln!("ok");
let post_state_success = if let Some(post_state) = blockchain.post_state.clone() {
check_poststate(&client, &name, post_state)
} else {
true
};
fail_unless(
client.chain_info().best_block_hash == blockchain.best_block.into()
&& post_state_success,
);
}
}
if fail {
flushln!(" - chain: {}...FAILED", name);
} else {
flushln!(" - chain: {}...OK", name);
}
start_stop_hook(&name, HookType::OnStop);
}
println!("!!! {:?} tests from failed.", failed.len());
failed
}
#[cfg(test)]
mod block_tests {
use super::json_chain_test;
use json_tests::HookType;
fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
json_chain_test(json_data, h)
}
declare_test! {BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"}
declare_test! {BlockchainTests_bcExploitTest, "BlockchainTests/bcExploitTest"}
declare_test! {BlockchainTests_bcForgedTest, "BlockchainTests/bcForgedTest"}
declare_test! {BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"}
declare_test! {BlockchainTests_bcGasPricerTest, "BlockchainTests/bcGasPricerTest"}
declare_test! {BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"}
declare_test! {BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"}
declare_test! {BlockchainTests_bcRandomBlockhashTest, "BlockchainTests/bcRandomBlockhashTest"}
declare_test! {BlockchainTests_bcStateTest, "BlockchainTests/bcStateTests"}
declare_test! {BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"}
declare_test! {BlockchainTests_bcUncleHeaderValidity, "BlockchainTests/bcUncleHeaderValidity"}
declare_test! {BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"}
declare_test! {BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"}
declare_test! {BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"}
declare_test! {BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"}
declare_test! {BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"}
declare_test! {BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"}
declare_test! {BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"}
declare_test! {BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"}
declare_test! {BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"}
declare_test! {BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"}
declare_test! {BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"}
declare_test! {BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"}
declare_test! {BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"}
declare_test! {BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"}
declare_test! {BlockchainTests_GeneralStateTest_stEIP150Specific, "BlockchainTests/GeneralStateTests/stEIP150Specific/"}
declare_test! {BlockchainTests_GeneralStateTest_stEIP158Specific, "BlockchainTests/GeneralStateTests/stEIP158Specific/"}
declare_test! {BlockchainTests_GeneralStateTest_stExample, "BlockchainTests/GeneralStateTests/stExample/"}
declare_test! {BlockchainTests_GeneralStateTest_stHomesteadSpecific, "BlockchainTests/GeneralStateTests/stHomesteadSpecific/"}
declare_test! {BlockchainTests_GeneralStateTest_stInitCodeTest, "BlockchainTests/GeneralStateTests/stInitCodeTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stLogTests, "BlockchainTests/GeneralStateTests/stLogTests/"}
declare_test! {BlockchainTests_GeneralStateTest_stMemExpandingEIP150Calls, "BlockchainTests/GeneralStateTests/stMemExpandingEIP150Calls/"}
declare_test! {heavy => BlockchainTests_GeneralStateTest_stMemoryStressTest, "BlockchainTests/GeneralStateTests/stMemoryStressTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"}
declare_test! {BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"}
declare_test! {heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"}
declare_test! {BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"}
declare_test! {BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"}
declare_test! {BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"}
declare_test! {BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"}
declare_test! {BlockchainTests_GeneralStateTest_stStaticCall, "BlockchainTests/GeneralStateTests/stStaticCall/"}
declare_test! {BlockchainTests_GeneralStateTest_stSystemOperationsTest, "BlockchainTests/GeneralStateTests/stSystemOperationsTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stTransactionTest, "BlockchainTests/GeneralStateTests/stTransactionTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stTransitionTest, "BlockchainTests/GeneralStateTests/stTransitionTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stWalletTest, "BlockchainTests/GeneralStateTests/stWalletTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"}
declare_test! {BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"}
declare_test! {BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"}
declare_test! {BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"}
declare_test! {BlockchainTests_GeneralStateTest_stSStoreTest, "BlockchainTests/GeneralStateTests/stSStoreTest/"}
declare_test! {BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"}
declare_test! {BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"}
declare_test! {BlockchainTests_TransitionTests_bcHomesteadToDao, "BlockchainTests/TransitionTests/bcHomesteadToDao/"}
declare_test! {BlockchainTests_TransitionTests_bcHomesteadToEIP150, "BlockchainTests/TransitionTests/bcHomesteadToEIP150/"}
}

View File

@ -17,25 +17,28 @@
use ethereum_types::U256;
use ethjson;
use spec::Spec;
use std::path::Path;
use types::header::Header;
use super::HookType;
pub fn json_difficulty_test<H: FnMut(&str, HookType)>(
path: &Path,
json_data: &[u8],
spec: Spec,
start_stop_hook: &mut H,
) -> Vec<String> {
let _ = ::env_logger::try_init();
let tests = ethjson::test::DifficultyTest::load(json_data).unwrap();
let mut ret = Vec::new();
let _ = env_logger::try_init();
let tests = ethjson::test::DifficultyTest::load(json_data).expect(&format!(
"Could not parse JSON difficulty test data from {}",
path.display()
));
let engine = &spec.engine;
for (name, test) in tests.into_iter() {
start_stop_hook(&name, HookType::OnStart);
flush!(" - {}...", name);
println!(" - {}...", name);
let mut parent_header = Header::new();
let block_number: u64 = test.current_block_number.into();
parent_header.set_number(block_number - 1);
@ -48,65 +51,14 @@ pub fn json_difficulty_test<H: FnMut(&str, HookType)>(
header.set_timestamp(test.current_timestamp.into());
engine.populate_from_parent(&mut header, &parent_header);
let expected_difficulty: U256 = test.current_difficulty.into();
assert_eq!(header.difficulty(), &expected_difficulty);
flushln!("ok");
if header.difficulty() == &expected_difficulty {
flushln!(" - difficulty: {}...OK", name);
} else {
flushln!(" - difficulty: {}...FAILED", name);
ret.push(format!("{}:{}", path.to_string_lossy(), name));
}
start_stop_hook(&name, HookType::OnStop);
}
vec![]
}
macro_rules! difficulty_json_test {
( $spec:ident ) => {
use super::json_difficulty_test;
use json_tests::HookType;
use tempdir::TempDir;
fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
let tempdir = TempDir::new("").unwrap();
json_difficulty_test(json_data, ::ethereum::$spec(&tempdir.path()), h)
}
};
}
macro_rules! difficulty_json_test_nopath {
( $spec:ident ) => {
use super::json_difficulty_test;
use json_tests::HookType;
fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
json_difficulty_test(json_data, ::ethereum::$spec(), h)
}
};
}
mod difficulty_test {
difficulty_json_test!(new_foundation);
declare_test! {DifficultyTests_difficulty, "BasicTests/difficulty.json"}
}
mod difficulty_test_byzantium {
difficulty_json_test_nopath!(new_byzantium_test);
declare_test! {DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"}
}
mod difficulty_test_foundation {
difficulty_json_test!(new_foundation);
declare_test! {DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"}
}
// Disabling Ropsten diff tests; waiting for upstream ethereum/tests Constantinople update
//mod difficulty_test_ropsten {
// difficulty_json_test_nopath!(new_ropsten_test);
// declare_test!{DifficultyTests_difficultyRopsten, "BasicTests/difficultyRopsten.json"}
//}
mod difficulty_test_frontier {
difficulty_json_test_nopath!(new_frontier_test);
declare_test! {DifficultyTests_difficultyFrontier, "BasicTests/difficultyFrontier.json"}
}
mod difficulty_test_homestead {
difficulty_json_test_nopath!(new_homestead_test);
declare_test! {DifficultyTests_difficultyHomestead, "BasicTests/difficultyHomestead.json"}
ret
}

View File

@ -18,7 +18,7 @@ use super::test_common::*;
use bytes::Bytes;
use ethjson;
use ethtrie;
use evm::{Finalize, VMType};
use evm::Finalize;
use executive::*;
use externalities::*;
use hash::keccak;
@ -35,16 +35,6 @@ use vm::{
use super::HookType;
/// Run executive jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
}
/// Run executive jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, do_json_test, h)
}
#[derive(Debug, PartialEq, Clone)]
struct CallCreate {
data: Bytes,
@ -250,30 +240,25 @@ where
}
}
fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
let vms = VMType::all();
vms.iter()
.flat_map(|vm| do_json_test_for(vm, json_data, h))
.collect()
}
fn do_json_test_for<H: FnMut(&str, HookType)>(
vm_type: &VMType,
pub fn json_executive_test<H: FnMut(&str, HookType)>(
path: &Path,
json_data: &[u8],
start_stop_hook: &mut H,
) -> Vec<String> {
let tests = ethjson::vm::Test::load(json_data).unwrap();
let tests = ethjson::vm::Test::load(json_data).expect(&format!(
"Could not parse JSON executive test data from {}",
path.display()
));
let mut failed = Vec::new();
for (name, vm) in tests.into_iter() {
start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart);
start_stop_hook(&format!("{}", name), HookType::OnStart);
info!(target: "jsontests", "name: {:?}", name);
let mut fail = false;
let mut fail_unless = |cond: bool, s: &str| {
if !cond && !fail {
failed.push(format!("[{}] {}: {}", vm_type, name, s));
failed.push(format!("{}: {}", name, s));
fail = true
}
};
@ -362,27 +347,35 @@ fn do_json_test_for<H: FnMut(&str, HookType)>(
for (address, account) in vm.post_state.unwrap().into_iter() {
let address = address.into();
let code: Vec<u8> = account.code.into();
if let Some(code) = account.code {
let code: Vec<u8> = code.into();
let found_code = try_fail!(state.code(&address));
let found_balance = try_fail!(state.balance(&address));
let found_nonce = try_fail!(state.nonce(&address));
fail_unless(
found_code
.as_ref()
.map_or_else(|| code.is_empty(), |c| &**c == &code),
"code is incorrect",
);
fail_unless(
found_balance == account.balance.into(),
"balance is incorrect",
);
fail_unless(found_nonce == account.nonce.into(), "nonce is incorrect");
for (k, v) in account.storage {
}
let found_balance = try_fail!(state.balance(&address));
let found_nonce = try_fail!(state.nonce(&address));
if let Some(balance) = account.balance {
fail_unless(found_balance == balance.into(), "balance is incorrect");
}
if let Some(nonce) = account.nonce {
fail_unless(found_nonce == nonce.into(), "nonce is incorrect");
}
if let Some(storage) = account.storage {
for (k, v) in storage {
let key: U256 = k.into();
let value: U256 = v.into();
let found_storage = try_fail!(state.storage_at(&address, &From::from(key)));
fail_unless(found_storage == From::from(value), "storage is incorrect");
let found_storage =
try_fail!(state.storage_at(&address, &From::from(&key)));
fail_unless(
found_storage == From::from(&value),
"storage is incorrect",
);
}
}
}
@ -392,26 +385,14 @@ fn do_json_test_for<H: FnMut(&str, HookType)>(
}
};
start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop);
if fail {
println!(" - vm: {:?}...FAILED", name);
} else {
println!(" - vm: {:?}...OK", name);
}
for f in &failed {
error!("FAILED: {:?}", f);
start_stop_hook(&format!("{}", name), HookType::OnStop);
}
failed
}
declare_test! {ExecutiveTests_vmArithmeticTest, "VMTests/vmArithmeticTest"}
declare_test! {ExecutiveTests_vmBitwiseLogicOperationTest, "VMTests/vmBitwiseLogicOperation"}
declare_test! {ExecutiveTests_vmBlockInfoTest, "VMTests/vmBlockInfoTest"}
// TODO [todr] Fails with Signal 11 when using JIT
declare_test! {ExecutiveTests_vmEnvironmentalInfoTest, "VMTests/vmEnvironmentalInfo"}
declare_test! {ExecutiveTests_vmIOandFlowOperationsTest, "VMTests/vmIOandFlowOperations"}
declare_test! {ExecutiveTests_vmLogTest, "VMTests/vmLogTest"}
declare_test! {heavy => ExecutiveTests_vmPerformance, "VMTests/vmPerformance"}
declare_test! {ExecutiveTests_vmPushDupSwapTest, "VMTests/vmPushDupSwapTest"}
declare_test! {ExecutiveTests_vmRandomTest, "VMTests/vmRandomTest"}
declare_test! {ExecutiveTests_vmSha3Test, "VMTests/vmSha3Test"}
declare_test! {ExecutiveTests_vmSystemOperationsTest, "VMTests/vmSystemOperations"}
declare_test! {ExecutiveTests_vmTests, "VMTests/vmTests"}

View File

@ -16,35 +16,16 @@
//! Helpers and tests for operating on jsontests.
#[macro_use]
mod test_common;
mod chain;
mod difficulty;
mod executive;
mod skip;
pub mod runner;
mod state;
mod test_common;
mod transaction;
mod trie;
#[cfg(test)]
mod difficulty;
pub use self::test_common::HookType;
use self::skip::SKIP_TEST_STATE;
pub use self::{
chain::{run_test_file as run_chain_test_file, run_test_path as run_chain_test_path},
executive::{
run_test_file as run_executive_test_file, run_test_path as run_executive_test_path,
},
state::{run_test_file as run_state_test_file, run_test_path as run_state_test_path},
transaction::{
run_test_file as run_transaction_test_file, run_test_path as run_transaction_test_path,
},
trie::{
run_generic_test_file as run_generic_trie_test_file,
run_generic_test_path as run_generic_trie_test_path,
run_secure_test_file as run_secure_trie_test_file,
run_secure_test_path as run_secure_trie_test_path,
},
executive::json_executive_test,
test_common::{find_json_files_recursive, HookType},
};

View File

@ -0,0 +1,248 @@
use ethjson::test::{
ChainTests, DifficultyTests, EthereumTestSuite, ExecutiveTests, StateTests, TestChainSpec,
TestTrieSpec, TransactionTests, TrieTests,
};
use globset::Glob;
use log::info;
use rayon::prelude::*;
use std::path::{Path, PathBuf};
use tempfile::tempdir;
use trie::TrieSpec;
/// Result of tests execution
pub struct TestResult {
/// Number of success execution
pub success: usize,
/// Number of success execution
pub failed: Vec<String>,
}
impl TestResult {
/// Creates a new TestResult without results
pub fn zero() -> Self {
TestResult {
success: 0,
failed: Vec::new(),
}
}
/// Creates a new success TestResult
pub fn success() -> Self {
TestResult {
success: 1,
failed: Vec::new(),
}
}
/// Creates a new failed TestResult
pub fn failed(name: &str) -> Self {
TestResult {
success: 0,
failed: vec![name.to_string()],
}
}
}
impl std::ops::Add for TestResult {
type Output = Self;
fn add(self, other: Self) -> Self {
let mut mself = self;
mself.success += other.success;
mself.failed.extend_from_slice(&other.failed);
mself
}
}
impl std::ops::AddAssign for TestResult {
fn add_assign(&mut self, other: Self) {
self.success += other.success;
self.failed.extend_from_slice(&other.failed);
}
}
pub struct TestRunner(EthereumTestSuite);
impl TestRunner {
/// Loads a new JSON Test suite
pub fn load<R>(reader: R) -> Result<Self, serde_json::Error>
where
R: std::io::Read,
{
Ok(TestRunner(serde_json::from_reader(reader)?))
}
/// Run the tests with one thread
pub fn run_without_par(&self) -> TestResult {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(1)
.build()
.unwrap();
pool.install(|| self.run())
}
/// Run the tests
pub fn run(&self) -> TestResult {
let mut res = TestResult::zero();
for t in &self.0.chain {
res += Self::run_chain_tests(&t);
}
for t in &self.0.state {
res += Self::run_state_tests(&t);
}
for t in &self.0.difficulty {
res += Self::run_difficuly_tests(&t);
}
for t in &self.0.executive {
res += Self::run_executive_tests(&t);
}
for t in &self.0.transaction {
res += Self::run_transaction_tests(&t);
}
for t in &self.0.trie {
res += Self::run_trie_tests(&t);
}
res
}
fn run1<T, F>(test: &T, base_path: &PathBuf, f: F) -> TestResult
where
T: Send + Sync,
F: Fn(&T, &Path, &[u8]) -> Vec<String> + Send + Sync,
{
let result = super::find_json_files_recursive(&base_path)
.into_par_iter()
.map(|path| {
info!("{:?}", path);
let json = std::fs::read(&path).unwrap();
let faileds = f(test, &path, &json);
if faileds.len() > 0 {
TestResult::failed(&faileds.join(","))
} else {
TestResult::success()
}
})
.reduce(TestResult::zero, |a, b| a + b);
if result.success + result.failed.len() == 0 {
panic!("There is no tests in the specified path {:?}", base_path);
}
result
}
fn in_set(path: &Path, exprs: &[String]) -> bool {
for pathexp in exprs {
let glob = Glob::new(&pathexp)
.expect(&format!("cannot parse expression {}", pathexp))
.compile_matcher();
if glob.is_match(path) {
return true;
}
}
false
}
fn run_chain_tests(test: &ChainTests) -> TestResult {
Self::run1(
test,
&test.path,
|test: &ChainTests, path: &Path, json: &[u8]| {
for skip in &test.skip {
if Self::in_set(&path, &skip.paths) {
println!(" - {} ..SKIPPED", path.to_string_lossy());
return Vec::new();
}
}
super::chain::json_chain_test(&test, &path, &json, &mut |_, _| {})
},
)
}
fn run_state_tests(test: &StateTests) -> TestResult {
Self::run1(
test,
&test.path,
|test: &StateTests, path: &Path, json: &[u8]| {
for skip in &test.skip {
if Self::in_set(&path, &skip.paths) {
println!(" - {} ..SKIPPED", path.to_string_lossy());
return Vec::new();
}
}
super::state::json_chain_test(&test, &path, &json, &mut |_, _| {})
},
)
}
fn run_difficuly_tests(test: &DifficultyTests) -> TestResult {
let mut acc = TestResult::zero();
for path in &test.path {
acc += Self::run1(
test,
&path,
|test: &DifficultyTests, path: &Path, json: &[u8]| {
let spec = match &test.chainspec {
TestChainSpec::Foundation => {
crate::ethereum::new_foundation(&tempdir().unwrap().path())
}
TestChainSpec::ByzantiumTest => crate::ethereum::new_byzantium_test(),
TestChainSpec::FrontierTest => crate::ethereum::new_frontier_test(),
TestChainSpec::HomesteadTest => crate::ethereum::new_homestead_test(),
};
super::difficulty::json_difficulty_test(&path, &json, spec, &mut |_, _| {})
},
)
}
acc
}
fn run_executive_tests(test: &ExecutiveTests) -> TestResult {
Self::run1(
test,
&test.path,
|_: &ExecutiveTests, path: &Path, json: &[u8]| {
super::executive::json_executive_test(&path, &json, &mut |_, _| {})
},
)
}
fn run_transaction_tests(test: &TransactionTests) -> TestResult {
Self::run1(
test,
&test.path,
|_: &TransactionTests, path: &Path, json: &[u8]| {
super::transaction::json_transaction_test(&path, &json, &mut |_, _| {})
},
)
}
fn run_trie_tests(test: &TrieTests) -> TestResult {
let mut acc = TestResult::zero();
for path in &test.path {
acc += Self::run1(test, &path, |test: &TrieTests, path: &Path, json: &[u8]| {
let spec = match &test.triespec {
TestTrieSpec::Generic => TrieSpec::Generic,
TestTrieSpec::Secure => TrieSpec::Secure,
};
super::trie::json_trie_test(&path, &json, spec, &mut |_, _| {})
});
}
acc
}
}
#[test]
fn ethereum_json_tests() {
let content = std::fs::read("res/ethereum/runner/full.json")
.expect("cannot open ethereum tests spec file");
let runner =
TestRunner::load(content.as_slice()).expect("cannot load ethereum tests spec file");
println!("----------------------------------------------------");
let result = runner.run();
println!("----------------------------------------------------");
flushln!(
"SUCCESS: {} FAILED: {} {:?}",
result.success,
result.failed.len(),
result.failed
);
assert!(result.failed.len() == 0);
}

View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
use super::{test_common::*, HookType, SKIP_TEST_STATE};
use super::{test_common::*, HookType};
use client::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess};
use ethjson;
use pod_state::PodState;
@ -23,19 +23,16 @@ use trace;
use types::transaction::SignedTransaction;
use vm::EnvInfo;
/// Run state jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, json_chain_test, h)
}
/// Run state jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, json_chain_test, h)
}
fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
SKIP_TEST_STATE.state.iter().any(|state_test| {
if let Some(subtest) = state_test.subtests.get(subname) {
fn skip_test(
test: &ethjson::test::StateTests,
subname: &str,
chain: &String,
number: usize,
) -> bool {
trace!(target: "json-tests", "[state, skip_test] subname: '{}', chain: '{}', number: {}", subname, chain, number);
test.skip.iter().any(|state_test| {
if let Some(subtest) = state_test.names.get(subname) {
trace!(target: "json-tests", "[state, skip_test] Maybe skipping {:?}", subtest);
chain == &subtest.chain
&& (subtest.subnumbers[0] == "*"
|| subtest.subnumbers.contains(&number.to_string()))
@ -46,11 +43,16 @@ fn skip_test(subname: &str, chain: &String, number: usize) -> bool {
}
pub fn json_chain_test<H: FnMut(&str, HookType)>(
state_test: &ethjson::test::StateTests,
path: &Path,
json_data: &[u8],
start_stop_hook: &mut H,
) -> Vec<String> {
let _ = ::env_logger::try_init();
let tests = ethjson::state::test::Test::load(json_data).unwrap();
let tests = ethjson::state::test::Test::load(json_data).expect(&format!(
"Could not parse JSON state test data from {}",
path.display()
));
let mut failed = Vec::new();
for (name, test) in tests.into_iter() {
@ -66,18 +68,23 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
let spec = match EvmTestClient::spec_from_json(&spec_name) {
Some(spec) => spec,
None => {
println!(
" - {} | {:?} Ignoring tests because of missing spec",
name, spec_name
panic!(
"Unimplemented chainspec '{:?}' in test '{}'",
spec_name, name
);
continue;
}
};
for (i, state) in states.into_iter().enumerate() {
let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total);
if skip_test(&name, &spec.name, i + 1) {
println!("{} in skip list : SKIPPED", info);
let info = format!(
" - state: {} | {:?} ({}/{}) ...",
name,
spec_name,
i + 1,
total
);
if skip_test(&state_test, &name, &spec.name, i + 1) {
println!("{}: SKIPPED", info);
continue;
}
@ -134,74 +141,5 @@ pub fn json_chain_test<H: FnMut(&str, HookType)>(
start_stop_hook(&name, HookType::OnStop);
}
if !failed.is_empty() {
println!("!!! {:?} tests failed.", failed.len());
}
failed
}
#[cfg(test)]
mod state_tests {
use super::json_chain_test;
use json_tests::HookType;
fn do_json_test<H: FnMut(&str, HookType)>(json_data: &[u8], h: &mut H) -> Vec<String> {
json_chain_test(json_data, h)
}
declare_test! {GeneralStateTest_stArgsZeroOneBalance, "GeneralStateTests/stArgsZeroOneBalance/"}
declare_test! {GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"}
declare_test! {GeneralStateTest_stBadOpcodeTest, "GeneralStateTests/stBadOpcode/"}
declare_test! {GeneralStateTest_stBugs, "GeneralStateTests/stBugs/"}
declare_test! {GeneralStateTest_stCallCodes, "GeneralStateTests/stCallCodes/"}
declare_test! {GeneralStateTest_stCallCreateCallCodeTest, "GeneralStateTests/stCallCreateCallCodeTest/"}
declare_test! {GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"}
declare_test! {GeneralStateTest_stCallDelegateCodesHomestead, "GeneralStateTests/stCallDelegateCodesHomestead/"}
declare_test! {GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"}
declare_test! {GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"}
declare_test! {GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"}
declare_test! {GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"}
declare_test! {GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"}
declare_test! {GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"}
declare_test! {GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"}
declare_test! {GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"}
declare_test! {GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"}
declare_test! {GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"}
declare_test! {GeneralStateTest_stExample, "GeneralStateTests/stExample/"}
declare_test! {GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"}
declare_test! {GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"}
declare_test! {GeneralStateTest_stLogTests, "GeneralStateTests/stLogTests/"}
declare_test! {GeneralStateTest_stMemExpandingEIP150Calls, "GeneralStateTests/stMemExpandingEIP150Calls/"}
declare_test! {heavy => GeneralStateTest_stMemoryStressTest, "GeneralStateTests/stMemoryStressTest/"}
declare_test! {GeneralStateTest_stMemoryTest, "GeneralStateTests/stMemoryTest/"}
declare_test! {GeneralStateTest_stNonZeroCallsTest, "GeneralStateTests/stNonZeroCallsTest/"}
declare_test! {GeneralStateTest_stPreCompiledContracts, "GeneralStateTests/stPreCompiledContracts/"}
declare_test! {GeneralStateTest_stPreCompiledContracts2, "GeneralStateTests/stPreCompiledContracts2/"}
declare_test! {heavy => GeneralStateTest_stQuadraticComplexityTest, "GeneralStateTests/stQuadraticComplexityTest/"}
declare_test! {GeneralStateTest_stRandom, "GeneralStateTests/stRandom/"}
declare_test! {GeneralStateTest_stRandom2, "GeneralStateTests/stRandom2/"}
declare_test! {GeneralStateTest_stRecursiveCreate, "GeneralStateTests/stRecursiveCreate/"}
declare_test! {GeneralStateTest_stRefundTest, "GeneralStateTests/stRefundTest/"}
declare_test! {GeneralStateTest_stReturnDataTest, "GeneralStateTests/stReturnDataTest/"}
declare_test! {GeneralStateTest_stRevertTest, "GeneralStateTests/stRevertTest/"}
declare_test! {GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"}
declare_test! {GeneralStateTest_stShift, "GeneralStateTests/stShift/"}
declare_test! {GeneralStateTest_stSolidityTest, "GeneralStateTests/stSolidityTest/"}
declare_test! {GeneralStateTest_stSpecialTest, "GeneralStateTests/stSpecialTest/"}
declare_test! {GeneralStateTest_stStackTests, "GeneralStateTests/stStackTests/"}
declare_test! {GeneralStateTest_stStaticCall, "GeneralStateTests/stStaticCall/"}
declare_test! {GeneralStateTest_stSystemOperationsTest, "GeneralStateTests/stSystemOperationsTest/"}
declare_test! {GeneralStateTest_stTransactionTest, "GeneralStateTests/stTransactionTest/"}
declare_test! {GeneralStateTest_stTransitionTest, "GeneralStateTests/stTransitionTest/"}
declare_test! {GeneralStateTest_stWalletTest, "GeneralStateTests/stWalletTest/"}
declare_test! {GeneralStateTest_stZeroCallsRevert, "GeneralStateTests/stZeroCallsRevert/"}
declare_test! {GeneralStateTest_stZeroCallsTest, "GeneralStateTests/stZeroCallsTest/"}
declare_test! {GeneralStateTest_stZeroKnowledge, "GeneralStateTests/stZeroKnowledge/"}
// Attempts to send a transaction that requires more than current balance:
// Tx:
// https://github.com/ethereum/tests/blob/726b161ba8a739691006cc1ba080672bb50a9d49/GeneralStateTests/stZeroKnowledge2/ecmul_0-3_5616_28000_96.json#L170
// Balance:
// https://github.com/ethereum/tests/blob/726b161ba8a739691006cc1ba080672bb50a9d49/GeneralStateTests/stZeroKnowledge2/ecmul_0-3_5616_28000_96.json#L126
declare_test! {GeneralStateTest_stZeroKnowledge2, "GeneralStateTests/stZeroKnowledge2/"}
}

View File

@ -15,13 +15,8 @@
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
pub use ethereum_types::{Address, H256, U256};
use std::{
collections::HashSet,
ffi::OsString,
fs::{read_dir, File},
io::Read,
path::Path,
};
use std::path::PathBuf;
use walkdir::{DirEntry, WalkDir};
/// Indicate when to run the hook passed to test functions.
#[derive(Copy, Clone, Eq, PartialEq)]
@ -32,134 +27,11 @@ pub enum HookType {
OnStop,
}
pub fn run_test_path<H: FnMut(&str, HookType)>(
p: &Path,
skip: &[&'static str],
runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
start_stop_hook: &mut H,
) {
let mut errors = Vec::new();
run_test_path_inner(p, skip, runner, start_stop_hook, &mut errors);
let empty: [String; 0] = [];
assert_eq!(errors, empty);
}
fn run_test_path_inner<H: FnMut(&str, HookType)>(
p: &Path,
skip: &[&'static str],
runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
start_stop_hook: &mut H,
errors: &mut Vec<String>,
) {
let path = Path::new(p);
let s: HashSet<OsString> = skip
.iter()
.map(|s| {
let mut os: OsString = s.into();
os.push(".json");
os
})
.collect();
let extension = path.extension().and_then(|s| s.to_str());
if path.is_dir() {
for p in read_dir(path).unwrap().filter_map(|e| {
let e = e.unwrap();
if s.contains(&e.file_name()) {
None
} else {
Some(e.path())
}
}) {
run_test_path_inner(&p, skip, runner, start_stop_hook, errors);
}
} else if extension == Some("swp") || extension == None {
// Ignore junk
} else {
let mut path = p.to_path_buf();
path.set_extension("json");
run_test_file_append(&path, runner, start_stop_hook, errors)
}
}
fn run_test_file_append<H: FnMut(&str, HookType)>(
path: &Path,
runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
start_stop_hook: &mut H,
errors: &mut Vec<String>,
) {
let mut data = Vec::new();
let mut file = match File::open(&path) {
Ok(file) => file,
Err(_) => panic!("Error opening test file at: {:?}", path),
};
file.read_to_end(&mut data)
.expect("Error reading test file");
errors.append(&mut runner(&data, start_stop_hook));
}
pub fn run_test_file<H: FnMut(&str, HookType)>(
path: &Path,
runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec<String>,
start_stop_hook: &mut H,
) {
let mut data = Vec::new();
let mut file = match File::open(&path) {
Ok(file) => file,
Err(_) => panic!("Error opening test file at: {:?}", path),
};
file.read_to_end(&mut data)
.expect("Error reading test file");
let results = runner(&data, start_stop_hook);
let empty: [String; 0] = [];
assert_eq!(results, empty);
}
#[cfg(test)]
macro_rules! test {
($name: expr, $skip: expr) => {
::json_tests::test_common::run_test_path(
::std::path::Path::new(concat!("res/ethereum/tests/", $name)),
&$skip,
do_json_test,
&mut |_, _| (),
);
};
}
#[macro_export]
macro_rules! declare_test {
(skip => $arr: expr, $id: ident, $name: expr) => {
#[cfg(test)]
#[test]
#[allow(non_snake_case)]
fn $id() {
test!($name, $arr);
}
};
(ignore => $id: ident, $name: expr) => {
#[cfg(test)]
#[ignore]
#[test]
#[allow(non_snake_case)]
fn $id() {
test!($name, []);
}
};
(heavy => $id: ident, $name: expr) => {
#[cfg(test)]
#[cfg(feature = "test-heavy")]
#[test]
#[allow(non_snake_case)]
fn $id() {
test!($name, []);
}
};
($id: ident, $name: expr) => {
#[cfg(test)]
#[test]
#[allow(non_snake_case)]
fn $id() {
test!($name, []);
}
};
pub fn find_json_files_recursive(path: &PathBuf) -> Vec<PathBuf> {
WalkDir::new(path)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_name().to_string_lossy().ends_with(".json"))
.map(DirEntry::into_path)
.collect::<Vec<PathBuf>>()
}

View File

@ -22,37 +22,30 @@ use std::path::Path;
use transaction_ext::Transaction;
use types::{header::Header, transaction::UnverifiedTransaction};
/// Run transaction jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
}
/// Run transaction jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, do_json_test, h)
}
pub fn json_transaction_test<H: FnMut(&str, HookType)>(
path: &Path,
json_data: &[u8],
start_stop_hook: &mut H,
) -> Vec<String> {
// Block number used to run the tests.
// Make sure that all the specified features are activated.
const BLOCK_NUMBER: u64 = 0x6ffffffffffffe;
fn do_json_test<H: FnMut(&str, HookType)>(
json_data: &[u8],
start_stop_hook: &mut H,
) -> Vec<String> {
let tests = ethjson::transaction::Test::load(json_data).unwrap();
let tests = ethjson::transaction::Test::load(json_data).expect(&format!(
"Could not parse JSON transaction test data from {}",
path.display()
));
let mut failed = Vec::new();
for (name, test) in tests.into_iter() {
start_stop_hook(&name, HookType::OnStart);
println!(" - tx: {} ", name);
for (spec_name, result) in test.post_state {
let spec = match EvmTestClient::spec_from_json(&spec_name) {
Some(spec) => spec,
None => {
println!(
" - {} | {:?} Ignoring tests because of missing spec",
name, spec_name
);
failed.push(format!("{}-{:?} (missing spec)", name, spec_name));
continue;
}
};
@ -110,14 +103,3 @@ fn do_json_test<H: FnMut(&str, HookType)>(
}
failed
}
declare_test! {TransactionTests_ttAddress, "TransactionTests/ttAddress"}
declare_test! {TransactionTests_ttData, "TransactionTests/ttData"}
declare_test! {TransactionTests_ttGasLimit, "TransactionTests/ttGasLimit"}
declare_test! {TransactionTests_ttGasPrice, "TransactionTests/ttGasPrice"}
declare_test! {TransactionTests_ttNonce, "TransactionTests/ttNonce"}
declare_test! {TransactionTests_ttRSValue, "TransactionTests/ttRSValue"}
declare_test! {TransactionTests_ttSignature, "TransactionTests/ttSignature"}
declare_test! {TransactionTests_ttValue, "TransactionTests/ttValue"}
declare_test! {TransactionTests_ttVValue, "TransactionTests/ttVValue"}
declare_test! {TransactionTests_ttWrongRLP, "TransactionTests/ttWrongRLP"}

View File

@ -17,29 +17,29 @@
use ethereum_types::H256;
use ethjson;
use ethtrie::RlpCodec;
use std::path::Path;
use trie::{TrieFactory, TrieSpec};
use super::HookType;
pub use self::{
generic::{run_test_file as run_generic_test_file, run_test_path as run_generic_test_path},
secure::{run_test_file as run_secure_test_file, run_test_path as run_secure_test_path},
};
fn test_trie<H: FnMut(&str, HookType)>(
pub fn json_trie_test<H: FnMut(&str, HookType)>(
path: &Path,
json: &[u8],
trie: TrieSpec,
start_stop_hook: &mut H,
) -> Vec<String> {
let tests = ethjson::trie::Test::load(json).unwrap();
let tests = ethjson::trie::Test::load(json).expect(&format!(
"Could not parse JSON trie test data from {}",
path.display()
));
let factory = TrieFactory::<_, RlpCodec>::new(trie);
let mut result = vec![];
let mut failed = vec![];
for (name, test) in tests.into_iter() {
start_stop_hook(&name, HookType::OnStart);
let mut memdb = journaldb::new_memory_db();
let mut root = H256::default();
let mut root = H256::zero();
let mut t = factory.create(&mut memdb, &mut root);
for (key, value) in test.input.data.into_iter() {
@ -51,65 +51,14 @@ fn test_trie<H: FnMut(&str, HookType)>(
));
}
if *t.root() != test.root.into() {
result.push(format!("Trie test '{:?}' failed.", name));
if *t.root() == test.root.into() {
println!(" - trie: {}...OK", name);
} else {
println!(" - trie: {}...FAILED ({:?})", name, path);
failed.push(format!("{}", name));
}
start_stop_hook(&name, HookType::OnStop);
}
for i in &result {
println!("FAILED: {}", i);
}
result
}
mod generic {
use std::path::Path;
use trie::TrieSpec;
use super::HookType;
/// Run generic trie jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
}
/// Run generic trie jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, do_json_test, h)
}
fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
super::test_trie(json, TrieSpec::Generic, h)
}
declare_test! {TrieTests_trietest, "TrieTests/trietest"}
declare_test! {TrieTests_trieanyorder, "TrieTests/trieanyorder"}
}
mod secure {
use std::path::Path;
use trie::TrieSpec;
use super::HookType;
/// Run secure trie jsontests on a given folder.
pub fn run_test_path<H: FnMut(&str, HookType)>(p: &Path, skip: &[&'static str], h: &mut H) {
::json_tests::test_common::run_test_path(p, skip, do_json_test, h)
}
/// Run secure trie jsontests on a given file.
pub fn run_test_file<H: FnMut(&str, HookType)>(p: &Path, h: &mut H) {
::json_tests::test_common::run_test_file(p, do_json_test, h)
}
fn do_json_test<H: FnMut(&str, HookType)>(json: &[u8], h: &mut H) -> Vec<String> {
super::test_trie(json, TrieSpec::Secure, h)
}
declare_test! {TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"}
declare_test! {TrieTests_trietest_secure, "TrieTests/trietest_secureTrie"}
declare_test! {TrieTests_trieanyorder_secure, "TrieTests/trieanyorder_secureTrie"}
failed
}

View File

@ -112,6 +112,8 @@ extern crate env_logger;
extern crate ethcore_accounts as accounts;
#[cfg(feature = "stratum")]
extern crate ethcore_stratum;
#[cfg(feature = "json-tests")]
extern crate globset;
#[cfg(any(test, feature = "test-helpers"))]
extern crate kvdb_memorydb;
#[cfg(any(test, feature = "kvdb-rocksdb"))]
@ -120,6 +122,10 @@ extern crate kvdb_rocksdb;
extern crate rlp_compress;
#[cfg(any(test, feature = "tempdir"))]
extern crate tempdir;
#[cfg(feature = "json-tests")]
extern crate tempfile;
#[cfg(feature = "json-tests")]
extern crate walkdir;
#[macro_use]
extern crate ethabi_derive;

View File

@ -148,10 +148,9 @@ fn run_stats_jsontests_vm(args: Args) {
});
}
};
if !file.is_file() {
json_tests::run_executive_test_path(&file, &[], &mut record_time);
} else {
json_tests::run_executive_test_file(&file, &mut record_time);
for file_path in json_tests::find_json_files_recursive(&file) {
let json_data = std::fs::read(&file_path).unwrap();
json_tests::json_executive_test(&file_path, &json_data, &mut record_time);
}
}

View File

@ -49,7 +49,7 @@ pub struct BlockChain {
/// Blocks.
pub blocks: Vec<Block>,
/// Post state.
pub post_state: State,
pub post_state: Option<State>,
/// Pre state.
#[serde(rename = "pre")]
pub pre_state: State,

View File

@ -16,19 +16,62 @@
//! Blockchain test state deserializer.
use blockchain::account::Account;
use hash::Address;
use crate::{
bytes::Bytes,
hash::{Address, H256},
spec::{Account, Builtin},
};
use serde::Deserialize;
use std::collections::BTreeMap;
/// Blockchain test state deserializer.
#[derive(Debug, PartialEq, Deserialize, Clone)]
pub struct State(BTreeMap<Address, Account>);
#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(untagged)]
pub enum HashOrMap {
/// When the `postState` is large, tests sometimes just include the state root of the last
/// successful block here.
Hash(H256),
/// The expected `postState` of a test
Map(BTreeMap<Address, Account>),
}
/// Blockchain state deserializer.
#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct State(pub HashOrMap);
impl State {
/// Returns all builtins.
pub fn builtins(&self) -> BTreeMap<Address, Builtin> {
match &self.0 {
HashOrMap::Hash(_) => BTreeMap::default(),
HashOrMap::Map(map) => map
.iter()
.filter_map(|(add, ref acc)| acc.builtin.clone().map(|b| (add.clone(), b.into())))
.collect(),
}
}
/// Returns all constructors.
pub fn constructors(&self) -> BTreeMap<Address, Bytes> {
match &self.0 {
HashOrMap::Hash(_) => BTreeMap::default(),
HashOrMap::Map(map) => map
.iter()
.filter_map(|(add, ref acc)| acc.constructor.clone().map(|b| (add.clone(), b)))
.collect(),
}
}
}
impl IntoIterator for State {
type Item = <BTreeMap<Address, Account> as IntoIterator>::Item;
type IntoIter = <BTreeMap<Address, Account> as IntoIterator>::IntoIter;
fn into_iter(self) -> Self::IntoIter {
self.0.into_iter()
if let HashOrMap::Map(m) = self.0 {
m.into_iter()
} else {
BTreeMap::default().into_iter()
}
}
}

View File

@ -23,7 +23,7 @@ use spec::builtin::BuiltinCompat;
use uint::Uint;
/// Spec account.
#[derive(Debug, PartialEq, Deserialize)]
#[derive(Clone, Debug, PartialEq, Deserialize)]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
pub struct Account {

View File

@ -35,6 +35,7 @@ pub enum ForkSpec {
FrontierToHomesteadAt5,
HomesteadToDaoAt5,
HomesteadToEIP150At5,
ByzantiumToConstantinopleAt5,
}
/// Spec deserialization.

View File

@ -18,7 +18,7 @@
use hash::H256;
use serde_json::{self, Error};
use std::{collections::BTreeMap, io::Read};
use std::{collections::BTreeMap, io::Read, path::PathBuf};
use uint::Uint;
/// Blockchain test header deserializer.
@ -119,3 +119,123 @@ impl SkipStates {
}
}
}
/// Describes a github.com/ethereum/tests suite
#[derive(Debug, PartialEq, Deserialize)]
pub struct EthereumTestSuite {
/// Blockchain tests
pub chain: Vec<ChainTests>,
/// State tests
pub state: Vec<StateTests>,
/// Difficulty tests
pub difficulty: Vec<DifficultyTests>,
/// Executive tests
pub executive: Vec<ExecutiveTests>,
/// Transaction tests
pub transaction: Vec<TransactionTests>,
/// Trie tests
pub trie: Vec<TrieTests>,
}
/// Chain spec used in tests
#[derive(Debug, PartialEq, Deserialize)]
pub enum TestChainSpec {
/// Foundation
Foundation,
/// ByzantiumTest
ByzantiumTest,
/// FrontierTest
FrontierTest,
/// HomesteadTest
HomesteadTest,
}
/// Kind of trie used in test
#[derive(Debug, PartialEq, Deserialize)]
pub enum TestTrieSpec {
/// Generic
Generic,
/// Secure
Secure,
}
/// A set of blockchain tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct ChainTests {
/// Path of the json tests
pub path: PathBuf,
/// Tests to skip
pub skip: Vec<ChainTestSkip>,
}
/// Tests to skip in chain tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct ChainTestSkip {
/// Issue reference.
pub reference: String,
/// Test names to skip
pub names: Vec<String>,
/// Test paths to skip
pub paths: Vec<String>,
}
/// A set of state tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct StateTests {
/// Path of the json tests
pub path: PathBuf,
/// Tests to skip
pub skip: Vec<StateTestSkip>,
}
/// State test to skip
#[derive(Debug, PartialEq, Deserialize)]
pub struct StateTestSkip {
/// Issue reference.
pub reference: String,
/// Paths to skip
pub paths: Vec<String>,
/// Test names to skip
pub names: BTreeMap<String, StateSkipSubStates1>,
}
/// State subtest to skip.
#[derive(Debug, PartialEq, Deserialize)]
pub struct StateSkipSubStates1 {
/// State test number of this item. Or '*' for all state.
pub subnumbers: Vec<String>,
/// Chain for this items.
pub chain: String,
}
/// A set of difficulty tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct DifficultyTests {
/// Path of the json tests
pub path: Vec<PathBuf>,
/// Chain spec to use
pub chainspec: TestChainSpec,
}
/// A set of executive tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct ExecutiveTests {
/// Path of the json tests
pub path: PathBuf,
}
/// A set of transaction tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct TransactionTests {
/// Path of the json tests
pub path: PathBuf,
}
/// A set of trie tests
#[derive(Debug, PartialEq, Deserialize)]
pub struct TrieTests {
/// Path of the json tests
pub path: Vec<PathBuf>,
/// Trie spec to use
pub triespec: TestTrieSpec,
}