From 499da19d825d586ac3e82a97d8ff2304333cb726 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 18:56:36 +0100 Subject: [PATCH] Initial draft of blockchain tests. --- src/block_queue.rs | 7 +++++ src/blockchain.rs | 8 ----- src/client.rs | 22 +++++++++---- src/pod_state.rs | 12 ++++++-- src/spec.rs | 73 ++++++++++++++++++++++---------------------- src/state.rs | 2 +- src/state_diff.rs | 14 ++++----- src/tests/chain.rs | 49 +++++++++++++++++++++++++++++ src/tests/mod.rs | 1 + util/src/standard.rs | 9 +++--- 10 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 src/tests/chain.rs diff --git a/src/block_queue.rs b/src/block_queue.rs index 239c559c5..36539bfff 100644 --- a/src/block_queue.rs +++ b/src/block_queue.rs @@ -21,6 +21,11 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, } +impl BlockQueueInfo { + /// The total size of the queues. + pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size } +} + /// A queue of blocks. Sits between network or other I/O and the BlockChain. /// Sorts them ready for blockchain insertion. pub struct BlockQueue { @@ -99,6 +104,7 @@ impl BlockQueue { fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc) { while !deleting.load(AtomicOrdering::Relaxed) { + { let mut lock = verification.lock().unwrap(); while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { @@ -139,6 +145,7 @@ impl BlockQueue { }, Err(err) => { let mut v = verification.lock().unwrap(); + flushln!("Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); v.bad.insert(block_hash.clone()); v.verifying.retain(|e| e.hash != block_hash); diff --git a/src/blockchain.rs b/src/blockchain.rs index 39390de97..3bd31688a 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -283,13 +283,6 @@ impl BlockChain { bc } - /// Ensure that the best block does indeed have a state_root in the state DB. - /// If it doesn't, then rewind down until we find one that does and delete data to ensure that - /// later blocks will be reimported. - pub fn ensure_good(&mut self, _state: &JournalDB) { - unimplemented!(); - } - /// Returns a tree route between `from` and `to`, which is a tuple of: /// /// - a vector of hashes of all blocks, ordered from `from` to `to`. @@ -392,7 +385,6 @@ impl BlockChain { } } - /// Inserts the block into backing cache database. /// Expects the block to be valid and already verified. /// If the block is already known, does nothing. diff --git a/src/client.rs b/src/client.rs index 4461f3d7b..795bca546 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,3 +1,5 @@ +use std::thread; +use std::time; use util::*; use rocksdb::{Options, DB}; use blockchain::{BlockChain, BlockProvider, CacheSize}; @@ -121,6 +123,7 @@ impl ClientReport { } /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. +/// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue. pub struct Client { chain: Arc>, engine: Arc>, @@ -140,7 +143,8 @@ impl Client { let mut opts = Options::new(); opts.set_max_open_files(256); opts.create_if_missing(true); - /*opts.set_use_fsync(false); + opts.set_use_fsync(false); + /* opts.set_bytes_per_sync(8388608); opts.set_disable_data_sync(false); opts.set_block_cache_size_mb(1024); @@ -177,16 +181,22 @@ impl Client { })) } + /// Flush the block import queue. + pub fn flush_queue(&self) { + flushln!("Flushing queue {:?}", self.block_queue.read().unwrap().queue_info()); + while self.block_queue.read().unwrap().queue_info().unverified_queue_size > 0 { + thread::sleep(time::Duration::from_millis(20)); + flushln!("Flushing queue [waited] {:?}", self.block_queue.read().unwrap().queue_info()); + } + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, _io: &IoChannel) { let mut bad = HashSet::new(); let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.write().unwrap().drain(128); - if blocks.is_empty() { - return; - } - for block in blocks { + for block in self.block_queue.write().unwrap().drain(128) { + flushln!("Importing block..."); if bad.contains(&block.header.parent_hash) { self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); bad.insert(block.header.hash()); diff --git a/src/pod_state.rs b/src/pod_state.rs index 1ea8382a5..15ae6a8ae 100644 --- a/src/pod_state.rs +++ b/src/pod_state.rs @@ -1,17 +1,25 @@ use util::*; use pod_account::*; -#[derive(Debug,Clone,PartialEq,Eq)] +#[derive(Debug,Clone,PartialEq,Eq,Default)] /// TODO [Gav Wood] Please document me pub struct PodState (BTreeMap); impl PodState { /// Contruct a new object from the `m`. - pub fn new(m: BTreeMap) -> PodState { PodState(m) } + pub fn new() -> PodState { Default::default() } + + /// Contruct a new object from the `m`. + pub fn from(m: BTreeMap) -> PodState { PodState(m) } /// Get the underlying map. pub fn get(&self) -> &BTreeMap { &self.0 } + /// Get the root hash of the trie of the RLP of this. + pub fn root(&self) -> H256 { + sec_trie_root(self.0.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect()) + } + /// Drain object to get the underlying map. pub fn drain(self) -> BTreeMap { self.0 } } diff --git a/src/spec.rs b/src/spec.rs index 9f98d5e2a..9e97595b9 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -1,6 +1,7 @@ use common::*; use flate2::read::GzDecoder; use engine::*; +use pod_state::*; use null_engine::*; /// Converts file from base64 gzipped bytes to json @@ -40,28 +41,6 @@ fn json_to_rlp_map(json: &Json) -> HashMap { }) } -//TODO: add code and data -#[derive(Debug)] -/// Genesis account data. Does no thave a DB overlay cache -pub struct GenesisAccount { - // Balance of the account. - balance: U256, - // Nonce of the account. - nonce: U256, -} - -impl GenesisAccount { - /// TODO [arkpar] Please document me - pub fn rlp(&self) -> Bytes { - let mut stream = RlpStream::new_list(4); - stream.append(&self.nonce); - stream.append(&self.balance); - stream.append(&SHA3_NULL_RLP); - stream.append(&SHA3_EMPTY); - stream.out() - } -} - /// Parameters for a block chain; includes both those intrinsic to the design of the /// chain and those to be interpreted by the active chain engine. #[derive(Debug)] @@ -83,7 +62,7 @@ pub struct Spec { // Builtin-contracts are here for now but would like to abstract into Engine API eventually. /// TODO [Gav Wood] Please document me - pub builtins: HashMap, + pub builtins: BTreeMap, // Genesis params. /// TODO [Gav Wood] Please document me @@ -101,7 +80,7 @@ pub struct Spec { /// TODO [arkpar] Please document me pub extra_data: Bytes, /// TODO [Gav Wood] Please document me - pub genesis_state: HashMap, + pub genesis_state: PodState, /// TODO [Gav Wood] Please document me pub seal_fields: usize, /// TODO [Gav Wood] Please document me @@ -126,7 +105,7 @@ impl Spec { /// Return the state root for the genesis state, memoising accordingly. pub fn state_root(&self) -> H256 { if self.state_root_memo.read().unwrap().is_none() { - *self.state_root_memo.write().unwrap() = Some(sec_trie_root(self.genesis_state.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect())); + *self.state_root_memo.write().unwrap() = Some(self.genesis_state.root()); } self.state_root_memo.read().unwrap().as_ref().unwrap().clone() } @@ -174,6 +153,35 @@ impl Spec { ret.append_raw(&empty_list, 1); ret.out() } + + /// Overwrite the genesis components with the given JSON, assuming standard Ethereum test format. + pub fn overwrite_genesis(&mut self, genesis: &Json) { + let (seal_fields, seal_rlp) = { + if genesis.find("mixHash").is_some() && genesis.find("nonce").is_some() { + let mut s = RlpStream::new(); + s.append(&H256::from_json(&genesis["mixHash"])); + s.append(&H64::from_json(&genesis["nonce"])); + (2, s.out()) + } else { + // backup algo that will work with sealFields/sealRlp (and without). + ( + u64::from_json(&genesis["sealFields"]) as usize, + Bytes::from_json(&genesis["sealRlp"]) + ) + } + }; + + self.parent_hash = H256::from_json(&genesis["parentHash"]); + self.author = Address::from_json(&genesis["coinbase"]); + self.difficulty = U256::from_json(&genesis["difficulty"]); + self.gas_limit = U256::from_json(&genesis["gasLimit"]); + self.gas_used = U256::from_json(&genesis["gasUsed"]); + self.timestamp = u64::from_json(&genesis["timestamp"]); + self.extra_data = Bytes::from_json(&genesis["extraData"]); + self.seal_fields = seal_fields; + self.seal_rlp = seal_rlp; + self.state_root_memo = RwLock::new(genesis.find("stateRoot").and_then(|_| Some(H256::from_json(&genesis["stateRoot"])))); + } } impl FromJson for Spec { @@ -181,8 +189,8 @@ impl FromJson for Spec { fn from_json(json: &Json) -> Spec { // once we commit ourselves to some json parsing library (serde?) // move it to proper data structure - let mut state = HashMap::new(); - let mut builtins = HashMap::new(); + let mut builtins = BTreeMap::new(); + let mut state = PodState::new(); if let Some(&Json::Object(ref accounts)) = json.find("accounts") { for (address, acc) in accounts.iter() { @@ -192,15 +200,8 @@ impl FromJson for Spec { builtins.insert(addr.clone(), builtin); } } - let balance = acc.find("balance").and_then(|x| match *x { Json::String(ref b) => U256::from_dec_str(b).ok(), _ => None }); - let nonce = acc.find("nonce").and_then(|x| match *x { Json::String(ref b) => U256::from_dec_str(b).ok(), _ => None }); -// let balance = if let Some(&Json::String(ref b)) = acc.find("balance") {U256::from_dec_str(b).unwrap_or(U256::from(0))} else {U256::from(0)}; -// let nonce = if let Some(&Json::String(ref n)) = acc.find("nonce") {U256::from_dec_str(n).unwrap_or(U256::from(0))} else {U256::from(0)}; - // TODO: handle code & data if they exist. - if balance.is_some() || nonce.is_some() { - state.insert(addr, GenesisAccount { balance: balance.unwrap_or_else(U256::zero), nonce: nonce.unwrap_or_else(U256::zero) }); - } } + state = xjson!(&json["accounts"]); } let nodes = if let Some(&Json::Array(ref ns)) = json.find("nodes") { @@ -253,7 +254,7 @@ impl Spec { let mut root = H256::new(); { let mut t = SecTrieDBMut::new(db, &mut root); - for (address, account) in &self.genesis_state { + for (address, account) in self.genesis_state.get().iter() { t.insert(address.as_slice(), &account.rlp()); } } diff --git a/src/state.rs b/src/state.rs index 4b7e5af34..6e5d586f3 100644 --- a/src/state.rs +++ b/src/state.rs @@ -221,7 +221,7 @@ impl State { /// Populate a PodAccount map from this state. pub fn to_pod(&self) -> PodState { // TODO: handle database rather than just the cache. - PodState::new(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| { + PodState::from(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| { if let Some(ref acc) = *opt { m.insert(add.clone(), PodAccount::from_account(acc)); } diff --git a/src/state_diff.rs b/src/state_diff.rs index 08fccf3ed..12e2d76ca 100644 --- a/src/state_diff.rs +++ b/src/state_diff.rs @@ -32,8 +32,8 @@ mod test { #[test] fn create_delete() { - let a = PodState::new(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - assert_eq!(StateDiff::diff_pod(&a, &PodState::new(map![])), StateDiff(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); + assert_eq!(StateDiff::diff_pod(&a, &PodState::new()), StateDiff(map![ x!(1) => AccountDiff{ balance: Diff::Died(x!(69)), nonce: Diff::Died(x!(0)), @@ -41,7 +41,7 @@ mod test { storage: map![], } ])); - assert_eq!(StateDiff::diff_pod(&PodState::new(map![]), &a), StateDiff(map![ + assert_eq!(StateDiff::diff_pod(&PodState::new(), &a), StateDiff(map![ x!(1) => AccountDiff{ balance: Diff::Born(x!(69)), nonce: Diff::Born(x!(0)), @@ -53,8 +53,8 @@ mod test { #[test] fn create_delete_with_unchanged() { - let a = PodState::new(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - let b = PodState::new(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); + let b = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); @@ -78,11 +78,11 @@ mod test { #[test] fn change_with_unchanged() { - let a = PodState::new(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - let b = PodState::new(map![ + let b = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(1), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); diff --git a/src/tests/chain.rs b/src/tests/chain.rs new file mode 100644 index 000000000..db3e398b0 --- /dev/null +++ b/src/tests/chain.rs @@ -0,0 +1,49 @@ +use std::env; +use super::test_common::*; +use client::{BlockChainClient,Client}; +use pod_state::*; +use ethereum; + +fn do_json_test(json_data: &[u8]) -> Vec { + let json = Json::from_str(::std::str::from_utf8(json_data).unwrap()).expect("Json is invalid"); + let mut failed = Vec::new(); + + for (name, test) in json.as_object().unwrap() { + let mut fail = false; + { + let mut fail_unless = |cond: bool| if !cond && !fail { + failed.push(name.clone()); + flush(format!("FAIL\n")); + fail = true; + true + } else {false}; + + flush(format!(" - {}...", name)); + + let blocks: Vec = test["blocks"].as_array().unwrap().iter().map(|e| xjson!(&e["rlp"])).collect(); + let mut spec = ethereum::new_frontier_like_test(); + spec.overwrite_genesis(test.find("genesisBlockHeader").unwrap()); + spec.genesis_state = PodState::from_json(test.find("pre").unwrap()); + + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + { + let client = Client::new(spec, &dir, IoChannel::disconnected()).unwrap(); + blocks.into_iter().foreach(|b| { + client.import_block(b).unwrap(); + }); + client.flush_queue(); + client.import_verified_blocks(&IoChannel::disconnected()); + flushln!("Best hash: {}", client.chain_info().best_block_hash); + } + fs::remove_dir_all(&dir).unwrap(); + } + if !fail { + flush(format!("ok\n")); + } + } + println!("!!! {:?} tests from failed.", failed.len()); + failed +} + +declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index c30f7f9b8..799c47230 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -4,3 +4,4 @@ mod test_common; mod transaction; mod executive; mod state; +mod chain; diff --git a/util/src/standard.rs b/util/src/standard.rs index 19a084a2c..873df6cb4 100644 --- a/util/src/standard.rs +++ b/util/src/standard.rs @@ -1,13 +1,14 @@ pub use std::io; +pub use std::fs; pub use std::str; pub use std::fmt; -pub use std::slice; pub use std::cmp; pub use std::ptr; -pub use std::result; -pub use std::option; pub use std::mem; pub use std::ops; +pub use std::slice; +pub use std::result; +pub use std::option; pub use std::path::Path; pub use std::str::{FromStr}; @@ -15,9 +16,9 @@ pub use std::io::{Read,Write}; pub use std::hash::{Hash, Hasher}; pub use std::error::Error as StdError; -pub use std::sync::*; pub use std::ops::*; pub use std::cmp::*; +pub use std::sync::*; pub use std::cell::*; pub use std::collections::*;