From 499da19d825d586ac3e82a97d8ff2304333cb726 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 18:56:36 +0100 Subject: [PATCH 1/9] Initial draft of blockchain tests. --- src/block_queue.rs | 7 +++++ src/blockchain.rs | 8 ----- src/client.rs | 22 +++++++++---- src/pod_state.rs | 12 ++++++-- src/spec.rs | 73 ++++++++++++++++++++++---------------------- src/state.rs | 2 +- src/state_diff.rs | 14 ++++----- src/tests/chain.rs | 49 +++++++++++++++++++++++++++++ src/tests/mod.rs | 1 + util/src/standard.rs | 9 +++--- 10 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 src/tests/chain.rs diff --git a/src/block_queue.rs b/src/block_queue.rs index 239c559c5..36539bfff 100644 --- a/src/block_queue.rs +++ b/src/block_queue.rs @@ -21,6 +21,11 @@ pub struct BlockQueueInfo { pub verified_queue_size: usize, } +impl BlockQueueInfo { + /// The total size of the queues. + pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size } +} + /// A queue of blocks. Sits between network or other I/O and the BlockChain. /// Sorts them ready for blockchain insertion. pub struct BlockQueue { @@ -99,6 +104,7 @@ impl BlockQueue { fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc) { while !deleting.load(AtomicOrdering::Relaxed) { + { let mut lock = verification.lock().unwrap(); while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { @@ -139,6 +145,7 @@ impl BlockQueue { }, Err(err) => { let mut v = verification.lock().unwrap(); + flushln!("Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); v.bad.insert(block_hash.clone()); v.verifying.retain(|e| e.hash != block_hash); diff --git a/src/blockchain.rs b/src/blockchain.rs index 39390de97..3bd31688a 100644 --- a/src/blockchain.rs +++ b/src/blockchain.rs @@ -283,13 +283,6 @@ impl BlockChain { bc } - /// Ensure that the best block does indeed have a state_root in the state DB. - /// If it doesn't, then rewind down until we find one that does and delete data to ensure that - /// later blocks will be reimported. - pub fn ensure_good(&mut self, _state: &JournalDB) { - unimplemented!(); - } - /// Returns a tree route between `from` and `to`, which is a tuple of: /// /// - a vector of hashes of all blocks, ordered from `from` to `to`. @@ -392,7 +385,6 @@ impl BlockChain { } } - /// Inserts the block into backing cache database. /// Expects the block to be valid and already verified. /// If the block is already known, does nothing. diff --git a/src/client.rs b/src/client.rs index 4461f3d7b..795bca546 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,3 +1,5 @@ +use std::thread; +use std::time; use util::*; use rocksdb::{Options, DB}; use blockchain::{BlockChain, BlockProvider, CacheSize}; @@ -121,6 +123,7 @@ impl ClientReport { } /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. +/// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue. pub struct Client { chain: Arc>, engine: Arc>, @@ -140,7 +143,8 @@ impl Client { let mut opts = Options::new(); opts.set_max_open_files(256); opts.create_if_missing(true); - /*opts.set_use_fsync(false); + opts.set_use_fsync(false); + /* opts.set_bytes_per_sync(8388608); opts.set_disable_data_sync(false); opts.set_block_cache_size_mb(1024); @@ -177,16 +181,22 @@ impl Client { })) } + /// Flush the block import queue. + pub fn flush_queue(&self) { + flushln!("Flushing queue {:?}", self.block_queue.read().unwrap().queue_info()); + while self.block_queue.read().unwrap().queue_info().unverified_queue_size > 0 { + thread::sleep(time::Duration::from_millis(20)); + flushln!("Flushing queue [waited] {:?}", self.block_queue.read().unwrap().queue_info()); + } + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, _io: &IoChannel) { let mut bad = HashSet::new(); let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.write().unwrap().drain(128); - if blocks.is_empty() { - return; - } - for block in blocks { + for block in self.block_queue.write().unwrap().drain(128) { + flushln!("Importing block..."); if bad.contains(&block.header.parent_hash) { self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); bad.insert(block.header.hash()); diff --git a/src/pod_state.rs b/src/pod_state.rs index 1ea8382a5..15ae6a8ae 100644 --- a/src/pod_state.rs +++ b/src/pod_state.rs @@ -1,17 +1,25 @@ use util::*; use pod_account::*; -#[derive(Debug,Clone,PartialEq,Eq)] +#[derive(Debug,Clone,PartialEq,Eq,Default)] /// TODO [Gav Wood] Please document me pub struct PodState (BTreeMap); impl PodState { /// Contruct a new object from the `m`. - pub fn new(m: BTreeMap) -> PodState { PodState(m) } + pub fn new() -> PodState { Default::default() } + + /// Contruct a new object from the `m`. + pub fn from(m: BTreeMap) -> PodState { PodState(m) } /// Get the underlying map. pub fn get(&self) -> &BTreeMap { &self.0 } + /// Get the root hash of the trie of the RLP of this. + pub fn root(&self) -> H256 { + sec_trie_root(self.0.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect()) + } + /// Drain object to get the underlying map. pub fn drain(self) -> BTreeMap { self.0 } } diff --git a/src/spec.rs b/src/spec.rs index 9f98d5e2a..9e97595b9 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -1,6 +1,7 @@ use common::*; use flate2::read::GzDecoder; use engine::*; +use pod_state::*; use null_engine::*; /// Converts file from base64 gzipped bytes to json @@ -40,28 +41,6 @@ fn json_to_rlp_map(json: &Json) -> HashMap { }) } -//TODO: add code and data -#[derive(Debug)] -/// Genesis account data. Does no thave a DB overlay cache -pub struct GenesisAccount { - // Balance of the account. - balance: U256, - // Nonce of the account. - nonce: U256, -} - -impl GenesisAccount { - /// TODO [arkpar] Please document me - pub fn rlp(&self) -> Bytes { - let mut stream = RlpStream::new_list(4); - stream.append(&self.nonce); - stream.append(&self.balance); - stream.append(&SHA3_NULL_RLP); - stream.append(&SHA3_EMPTY); - stream.out() - } -} - /// Parameters for a block chain; includes both those intrinsic to the design of the /// chain and those to be interpreted by the active chain engine. #[derive(Debug)] @@ -83,7 +62,7 @@ pub struct Spec { // Builtin-contracts are here for now but would like to abstract into Engine API eventually. /// TODO [Gav Wood] Please document me - pub builtins: HashMap, + pub builtins: BTreeMap, // Genesis params. /// TODO [Gav Wood] Please document me @@ -101,7 +80,7 @@ pub struct Spec { /// TODO [arkpar] Please document me pub extra_data: Bytes, /// TODO [Gav Wood] Please document me - pub genesis_state: HashMap, + pub genesis_state: PodState, /// TODO [Gav Wood] Please document me pub seal_fields: usize, /// TODO [Gav Wood] Please document me @@ -126,7 +105,7 @@ impl Spec { /// Return the state root for the genesis state, memoising accordingly. pub fn state_root(&self) -> H256 { if self.state_root_memo.read().unwrap().is_none() { - *self.state_root_memo.write().unwrap() = Some(sec_trie_root(self.genesis_state.iter().map(|(k, v)| (k.to_vec(), v.rlp())).collect())); + *self.state_root_memo.write().unwrap() = Some(self.genesis_state.root()); } self.state_root_memo.read().unwrap().as_ref().unwrap().clone() } @@ -174,6 +153,35 @@ impl Spec { ret.append_raw(&empty_list, 1); ret.out() } + + /// Overwrite the genesis components with the given JSON, assuming standard Ethereum test format. + pub fn overwrite_genesis(&mut self, genesis: &Json) { + let (seal_fields, seal_rlp) = { + if genesis.find("mixHash").is_some() && genesis.find("nonce").is_some() { + let mut s = RlpStream::new(); + s.append(&H256::from_json(&genesis["mixHash"])); + s.append(&H64::from_json(&genesis["nonce"])); + (2, s.out()) + } else { + // backup algo that will work with sealFields/sealRlp (and without). + ( + u64::from_json(&genesis["sealFields"]) as usize, + Bytes::from_json(&genesis["sealRlp"]) + ) + } + }; + + self.parent_hash = H256::from_json(&genesis["parentHash"]); + self.author = Address::from_json(&genesis["coinbase"]); + self.difficulty = U256::from_json(&genesis["difficulty"]); + self.gas_limit = U256::from_json(&genesis["gasLimit"]); + self.gas_used = U256::from_json(&genesis["gasUsed"]); + self.timestamp = u64::from_json(&genesis["timestamp"]); + self.extra_data = Bytes::from_json(&genesis["extraData"]); + self.seal_fields = seal_fields; + self.seal_rlp = seal_rlp; + self.state_root_memo = RwLock::new(genesis.find("stateRoot").and_then(|_| Some(H256::from_json(&genesis["stateRoot"])))); + } } impl FromJson for Spec { @@ -181,8 +189,8 @@ impl FromJson for Spec { fn from_json(json: &Json) -> Spec { // once we commit ourselves to some json parsing library (serde?) // move it to proper data structure - let mut state = HashMap::new(); - let mut builtins = HashMap::new(); + let mut builtins = BTreeMap::new(); + let mut state = PodState::new(); if let Some(&Json::Object(ref accounts)) = json.find("accounts") { for (address, acc) in accounts.iter() { @@ -192,15 +200,8 @@ impl FromJson for Spec { builtins.insert(addr.clone(), builtin); } } - let balance = acc.find("balance").and_then(|x| match *x { Json::String(ref b) => U256::from_dec_str(b).ok(), _ => None }); - let nonce = acc.find("nonce").and_then(|x| match *x { Json::String(ref b) => U256::from_dec_str(b).ok(), _ => None }); -// let balance = if let Some(&Json::String(ref b)) = acc.find("balance") {U256::from_dec_str(b).unwrap_or(U256::from(0))} else {U256::from(0)}; -// let nonce = if let Some(&Json::String(ref n)) = acc.find("nonce") {U256::from_dec_str(n).unwrap_or(U256::from(0))} else {U256::from(0)}; - // TODO: handle code & data if they exist. - if balance.is_some() || nonce.is_some() { - state.insert(addr, GenesisAccount { balance: balance.unwrap_or_else(U256::zero), nonce: nonce.unwrap_or_else(U256::zero) }); - } } + state = xjson!(&json["accounts"]); } let nodes = if let Some(&Json::Array(ref ns)) = json.find("nodes") { @@ -253,7 +254,7 @@ impl Spec { let mut root = H256::new(); { let mut t = SecTrieDBMut::new(db, &mut root); - for (address, account) in &self.genesis_state { + for (address, account) in self.genesis_state.get().iter() { t.insert(address.as_slice(), &account.rlp()); } } diff --git a/src/state.rs b/src/state.rs index 4b7e5af34..6e5d586f3 100644 --- a/src/state.rs +++ b/src/state.rs @@ -221,7 +221,7 @@ impl State { /// Populate a PodAccount map from this state. pub fn to_pod(&self) -> PodState { // TODO: handle database rather than just the cache. - PodState::new(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| { + PodState::from(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| { if let Some(ref acc) = *opt { m.insert(add.clone(), PodAccount::from_account(acc)); } diff --git a/src/state_diff.rs b/src/state_diff.rs index 08fccf3ed..12e2d76ca 100644 --- a/src/state_diff.rs +++ b/src/state_diff.rs @@ -32,8 +32,8 @@ mod test { #[test] fn create_delete() { - let a = PodState::new(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - assert_eq!(StateDiff::diff_pod(&a, &PodState::new(map![])), StateDiff(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); + assert_eq!(StateDiff::diff_pod(&a, &PodState::new()), StateDiff(map![ x!(1) => AccountDiff{ balance: Diff::Died(x!(69)), nonce: Diff::Died(x!(0)), @@ -41,7 +41,7 @@ mod test { storage: map![], } ])); - assert_eq!(StateDiff::diff_pod(&PodState::new(map![]), &a), StateDiff(map![ + assert_eq!(StateDiff::diff_pod(&PodState::new(), &a), StateDiff(map![ x!(1) => AccountDiff{ balance: Diff::Born(x!(69)), nonce: Diff::Born(x!(0)), @@ -53,8 +53,8 @@ mod test { #[test] fn create_delete_with_unchanged() { - let a = PodState::new(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - let b = PodState::new(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); + let b = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); @@ -78,11 +78,11 @@ mod test { #[test] fn change_with_unchanged() { - let a = PodState::new(map![ + let a = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(0), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); - let b = PodState::new(map![ + let b = PodState::from(map![ x!(1) => PodAccount::new(x!(69), x!(1), vec![], map![]), x!(2) => PodAccount::new(x!(69), x!(0), vec![], map![]) ]); diff --git a/src/tests/chain.rs b/src/tests/chain.rs new file mode 100644 index 000000000..db3e398b0 --- /dev/null +++ b/src/tests/chain.rs @@ -0,0 +1,49 @@ +use std::env; +use super::test_common::*; +use client::{BlockChainClient,Client}; +use pod_state::*; +use ethereum; + +fn do_json_test(json_data: &[u8]) -> Vec { + let json = Json::from_str(::std::str::from_utf8(json_data).unwrap()).expect("Json is invalid"); + let mut failed = Vec::new(); + + for (name, test) in json.as_object().unwrap() { + let mut fail = false; + { + let mut fail_unless = |cond: bool| if !cond && !fail { + failed.push(name.clone()); + flush(format!("FAIL\n")); + fail = true; + true + } else {false}; + + flush(format!(" - {}...", name)); + + let blocks: Vec = test["blocks"].as_array().unwrap().iter().map(|e| xjson!(&e["rlp"])).collect(); + let mut spec = ethereum::new_frontier_like_test(); + spec.overwrite_genesis(test.find("genesisBlockHeader").unwrap()); + spec.genesis_state = PodState::from_json(test.find("pre").unwrap()); + + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + { + let client = Client::new(spec, &dir, IoChannel::disconnected()).unwrap(); + blocks.into_iter().foreach(|b| { + client.import_block(b).unwrap(); + }); + client.flush_queue(); + client.import_verified_blocks(&IoChannel::disconnected()); + flushln!("Best hash: {}", client.chain_info().best_block_hash); + } + fs::remove_dir_all(&dir).unwrap(); + } + if !fail { + flush(format!("ok\n")); + } + } + println!("!!! {:?} tests from failed.", failed.len()); + failed +} + +declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index c30f7f9b8..799c47230 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -4,3 +4,4 @@ mod test_common; mod transaction; mod executive; mod state; +mod chain; diff --git a/util/src/standard.rs b/util/src/standard.rs index 19a084a2c..873df6cb4 100644 --- a/util/src/standard.rs +++ b/util/src/standard.rs @@ -1,13 +1,14 @@ pub use std::io; +pub use std::fs; pub use std::str; pub use std::fmt; -pub use std::slice; pub use std::cmp; pub use std::ptr; -pub use std::result; -pub use std::option; pub use std::mem; pub use std::ops; +pub use std::slice; +pub use std::result; +pub use std::option; pub use std::path::Path; pub use std::str::{FromStr}; @@ -15,9 +16,9 @@ pub use std::io::{Read,Write}; pub use std::hash::{Hash, Hasher}; pub use std::error::Error as StdError; -pub use std::sync::*; pub use std::ops::*; pub use std::cmp::*; +pub use std::sync::*; pub use std::cell::*; pub use std::collections::*; From a43ca9ae34d7dcd299cd82bc1ee36ac15464f9bb Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 25 Jan 2016 19:20:34 +0100 Subject: [PATCH 2/9] blockqueue flush --- src/block_queue.rs | 22 +++++++++++++++++++--- src/client.rs | 7 +------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/block_queue.rs b/src/block_queue.rs index 36539bfff..a0e46193b 100644 --- a/src/block_queue.rs +++ b/src/block_queue.rs @@ -35,6 +35,7 @@ pub struct BlockQueue { verifiers: Vec>, deleting: Arc, ready_signal: Arc, + empty: Arc, processing: HashSet } @@ -79,6 +80,7 @@ impl BlockQueue { let more_to_verify = Arc::new(Condvar::new()); let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); let deleting = Arc::new(AtomicBool::new(false)); + let empty = Arc::new(Condvar::new()); let mut verifiers: Vec> = Vec::new(); let thread_count = max(::num_cpus::get(), 3) - 2; @@ -87,8 +89,9 @@ impl BlockQueue { let engine = engine.clone(); let more_to_verify = more_to_verify.clone(); let ready_signal = ready_signal.clone(); + let empty = empty.clone(); let deleting = deleting.clone(); - verifiers.push(thread::Builder::new().name(format!("Verifier #{}", i)).spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting)) + verifiers.push(thread::Builder::new().name(format!("Verifier #{}", i)).spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)) .expect("Error starting block verification thread")); } BlockQueue { @@ -99,14 +102,20 @@ impl BlockQueue { verifiers: verifiers, deleting: deleting.clone(), processing: HashSet::new(), + empty: empty.clone(), } } - fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc) { + fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { while !deleting.load(AtomicOrdering::Relaxed) { - { let mut lock = verification.lock().unwrap(); + + if lock.unverified.is_empty() && lock.verifying.is_empty() { + empty.notify_all(); + } + + while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { lock = wait.wait(lock).unwrap(); } @@ -176,6 +185,13 @@ impl BlockQueue { verification.verifying.clear(); } + /// Wait for queue to be empty + pub fn flush(&mut self) { + let mutex: Mutex<()> = Mutex::new(()); + let lock = mutex.lock().unwrap(); + let _ = self.empty.wait(lock).unwrap(); + } + /// Add a block to the queue. pub fn import_block(&mut self, bytes: Bytes) -> ImportResult { let header = BlockView::new(&bytes).header(); diff --git a/src/client.rs b/src/client.rs index 795bca546..f05819370 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,5 +1,3 @@ -use std::thread; -use std::time; use util::*; use rocksdb::{Options, DB}; use blockchain::{BlockChain, BlockProvider, CacheSize}; @@ -184,10 +182,7 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { flushln!("Flushing queue {:?}", self.block_queue.read().unwrap().queue_info()); - while self.block_queue.read().unwrap().queue_info().unverified_queue_size > 0 { - thread::sleep(time::Duration::from_millis(20)); - flushln!("Flushing queue [waited] {:?}", self.block_queue.read().unwrap().queue_info()); - } + self.block_queue.write().unwrap().flush(); } /// This is triggered by a message coming from a block queue when the block is ready for insertion From 41508cbd507c4514be48cae43c1eb8d8cef4ab21 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 23:24:51 +0100 Subject: [PATCH 3/9] Fix queue flush and add working tests. --- src/block.rs | 1 + src/block_queue.rs | 19 ++++++++++++------- src/client.rs | 7 +++---- src/spec.rs | 13 ++++++++++++- src/state.rs | 6 +++--- src/sync/tests.rs | 1 + src/tests/chain.rs | 11 ++++++++--- 7 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/block.rs b/src/block.rs index 1ff326430..c63f49bd7 100644 --- a/src/block.rs +++ b/src/block.rs @@ -188,6 +188,7 @@ impl<'x, 'y> OpenBlock<'x, 'y> { // info!("env_info says gas_used={}", env_info.gas_used); match self.block.state.apply(&env_info, self.engine, &t) { Ok(receipt) => { + flushln!("Transaction executed {:?}", receipt); self.block.archive_set.insert(h.unwrap_or_else(||t.hash())); self.block.archive.push(Entry { transaction: t, receipt: receipt }); Ok(&self.block.archive.last().unwrap().receipt) diff --git a/src/block_queue.rs b/src/block_queue.rs index a0e46193b..fa091d0c4 100644 --- a/src/block_queue.rs +++ b/src/block_queue.rs @@ -19,11 +19,16 @@ pub struct BlockQueueInfo { pub unverified_queue_size: usize, /// Number of verified queued blocks pending import pub verified_queue_size: usize, + /// Number of blocks being verified + pub verifying_queue_size: usize, } impl BlockQueueInfo { /// The total size of the queues. - pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size } + pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size } + + /// The size of the unverified and verifying queues. + pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size } } /// A queue of blocks. Sits between network or other I/O and the BlockChain. @@ -91,7 +96,7 @@ impl BlockQueue { let ready_signal = ready_signal.clone(); let empty = empty.clone(); let deleting = deleting.clone(); - verifiers.push(thread::Builder::new().name(format!("Verifier #{}", i)).spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)) + verifiers.push(thread::Builder::new().name(format!("Verifier #{}", i)).spawn(move || BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty)) .expect("Error starting block verification thread")); } BlockQueue { @@ -115,7 +120,6 @@ impl BlockQueue { empty.notify_all(); } - while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Relaxed) { lock = wait.wait(lock).unwrap(); } @@ -154,7 +158,6 @@ impl BlockQueue { }, Err(err) => { let mut v = verification.lock().unwrap(); - flushln!("Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); v.bad.insert(block_hash.clone()); v.verifying.retain(|e| e.hash != block_hash); @@ -187,9 +190,10 @@ impl BlockQueue { /// Wait for queue to be empty pub fn flush(&mut self) { - let mutex: Mutex<()> = Mutex::new(()); - let lock = mutex.lock().unwrap(); - let _ = self.empty.wait(lock).unwrap(); + let mut verification = self.verification.lock().unwrap(); + while !verification.unverified.is_empty() && !verification.verifying.is_empty() { + verification = self.empty.wait(verification).unwrap(); + } } /// Add a block to the queue. @@ -265,6 +269,7 @@ impl BlockQueue { full: false, verified_queue_size: verification.verified.len(), unverified_queue_size: verification.unverified.len(), + verifying_queue_size: verification.verifying.len(), } } } diff --git a/src/client.rs b/src/client.rs index f05819370..2cc6c150d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -181,7 +181,6 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { - flushln!("Flushing queue {:?}", self.block_queue.read().unwrap().queue_info()); self.block_queue.write().unwrap().flush(); } @@ -189,9 +188,8 @@ impl Client { pub fn import_verified_blocks(&self, _io: &IoChannel) { let mut bad = HashSet::new(); let _import_lock = self.import_lock.lock(); - - for block in self.block_queue.write().unwrap().drain(128) { - flushln!("Importing block..."); + let blocks = self.block_queue.write().unwrap().drain(128); + for block in blocks { if bad.contains(&block.header.parent_hash) { self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); bad.insert(block.header.hash()); @@ -238,6 +236,7 @@ impl Client { } }; if let Err(e) = verify_block_final(&header, result.block().header()) { + flushln!("Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); self.block_queue.write().unwrap().mark_as_bad(&header.hash()); return; diff --git a/src/spec.rs b/src/spec.rs index 9e97595b9..326552524 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -80,7 +80,7 @@ pub struct Spec { /// TODO [arkpar] Please document me pub extra_data: Bytes, /// TODO [Gav Wood] Please document me - pub genesis_state: PodState, + genesis_state: PodState, /// TODO [Gav Wood] Please document me pub seal_fields: usize, /// TODO [Gav Wood] Please document me @@ -182,6 +182,17 @@ impl Spec { self.seal_rlp = seal_rlp; self.state_root_memo = RwLock::new(genesis.find("stateRoot").and_then(|_| Some(H256::from_json(&genesis["stateRoot"])))); } + + /// Alter the value of the genesis state. + pub fn set_genesis_state(&mut self, s: PodState) { + self.genesis_state = s; + *self.state_root_memo.write().unwrap() = None; + } + + /// Returns `false` if the memoized state root is invalid. `true` otherwise. + pub fn is_state_root_valid(&self) -> bool { + self.state_root_memo.read().unwrap().clone().map_or(true, |sr| sr == self.genesis_state.root()) + } } impl FromJson for Spec { diff --git a/src/state.rs b/src/state.rs index 6e5d586f3..71ae0ecda 100644 --- a/src/state.rs +++ b/src/state.rs @@ -146,15 +146,15 @@ impl State { /// This will change the state accordingly. pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, t: &Transaction) -> ApplyResult { - let old = self.to_pod(); +// let old = self.to_pod(); let e = try!(Executive::new(self, env_info, engine).transact(t)); //println!("Executed: {:?}", e); - trace!("Applied transaction. Diff:\n{}\n", StateDiff::diff_pod(&old, &self.to_pod())); +// trace!("Applied transaction. Diff:\n{}\n", StateDiff::diff_pod(&old, &self.to_pod())); self.commit(); let receipt = Receipt::new(self.root().clone(), e.cumulative_gas_used, e.logs); - trace!("Transaction receipt: {:?}", receipt); +// trace!("Transaction receipt: {:?}", receipt); Ok(receipt) } diff --git a/src/sync/tests.rs b/src/sync/tests.rs index 7f8a1748b..50d6efab2 100644 --- a/src/sync/tests.rs +++ b/src/sync/tests.rs @@ -158,6 +158,7 @@ impl BlockChainClient for TestBlockChainClient { full: false, verified_queue_size: 0, unverified_queue_size: 0, + verifying_queue_size: 0, } } diff --git a/src/tests/chain.rs b/src/tests/chain.rs index db3e398b0..922def6c7 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -22,8 +22,9 @@ fn do_json_test(json_data: &[u8]) -> Vec { let blocks: Vec = test["blocks"].as_array().unwrap().iter().map(|e| xjson!(&e["rlp"])).collect(); let mut spec = ethereum::new_frontier_like_test(); + spec.set_genesis_state(PodState::from_json(test.find("pre").unwrap())); spec.overwrite_genesis(test.find("genesisBlockHeader").unwrap()); - spec.genesis_state = PodState::from_json(test.find("pre").unwrap()); + assert!(spec.is_state_root_valid()); let mut dir = env::temp_dir(); dir.push(H32::random().hex()); @@ -32,9 +33,12 @@ fn do_json_test(json_data: &[u8]) -> Vec { blocks.into_iter().foreach(|b| { client.import_block(b).unwrap(); }); + flushln!("Imported all"); client.flush_queue(); + flushln!("Flushed"); client.import_verified_blocks(&IoChannel::disconnected()); - flushln!("Best hash: {}", client.chain_info().best_block_hash); + flushln!("Checking..."); + fail_unless(client.chain_info().best_block_hash == H256::from_json(&test["lastblockhash"])); } fs::remove_dir_all(&dir).unwrap(); } @@ -46,4 +50,5 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } -declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} +//declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} +declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} From 60af30558c91363239363aef632a41903a5a9cec Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 23:26:42 +0100 Subject: [PATCH 4/9] Cleanups. --- src/tests/chain.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tests/chain.rs b/src/tests/chain.rs index 922def6c7..aeb92d3fd 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -33,11 +33,8 @@ fn do_json_test(json_data: &[u8]) -> Vec { blocks.into_iter().foreach(|b| { client.import_block(b).unwrap(); }); - flushln!("Imported all"); client.flush_queue(); - flushln!("Flushed"); client.import_verified_blocks(&IoChannel::disconnected()); - flushln!("Checking..."); fail_unless(client.chain_info().best_block_hash == H256::from_json(&test["lastblockhash"])); } fs::remove_dir_all(&dir).unwrap(); @@ -50,5 +47,7 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } +// Fails. TODO: figure out why. //declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} + declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} From e479e8ca973ccd9e918518dbff5c7f6cf7e78785 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 23:37:49 +0100 Subject: [PATCH 5/9] Tody ups. --- src/state.rs | 4 ++-- src/tests/chain.rs | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/state.rs b/src/state.rs index 71ae0ecda..7310f63a7 100644 --- a/src/state.rs +++ b/src/state.rs @@ -3,7 +3,7 @@ use engine::Engine; use executive::Executive; use pod_account::*; use pod_state::*; -use state_diff::*; +//use state_diff::*; // TODO: uncomment once to_pod() works correctly. /// TODO [Gav Wood] Please document me pub type ApplyResult = Result; @@ -145,12 +145,12 @@ impl State { /// Execute a given transaction. /// This will change the state accordingly. pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, t: &Transaction) -> ApplyResult { - // let old = self.to_pod(); let e = try!(Executive::new(self, env_info, engine).transact(t)); //println!("Executed: {:?}", e); + // TODO uncomment once to_pod() works correctly. // trace!("Applied transaction. Diff:\n{}\n", StateDiff::diff_pod(&old, &self.to_pod())); self.commit(); let receipt = Receipt::new(self.root().clone(), e.cumulative_gas_used, e.logs); diff --git a/src/tests/chain.rs b/src/tests/chain.rs index aeb92d3fd..89199feed 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -47,7 +47,18 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } -// Fails. TODO: figure out why. -//declare_test!{BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} - +declare_test!{ignore => BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} // UNKNOWN declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} +declare_test!{ignore => BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcForkUncle, "BlockchainTests/bcForkUncle"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcGasPricerTest, "BlockchainTests/bcGasPricerTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcInvalidRLPTest, "BlockchainTests/bcInvalidRLPTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcRPC_API_Test, "BlockchainTests/bcRPC_API_Test"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} // FAILS (Suicides, GasUsed) +declare_test!{ignore => BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcUncleHeaderValiditiy, "BlockchainTests/bcUncleHeaderValiditiy"}// UNKNOWN +declare_test!{ignore => BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} // UNKNOWN From b6622e6efed4a68a70a28879261bdb7d475c640d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 25 Jan 2016 23:39:21 +0100 Subject: [PATCH 6/9] Remove flushln!s. --- src/block.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/block.rs b/src/block.rs index c63f49bd7..1ff326430 100644 --- a/src/block.rs +++ b/src/block.rs @@ -188,7 +188,6 @@ impl<'x, 'y> OpenBlock<'x, 'y> { // info!("env_info says gas_used={}", env_info.gas_used); match self.block.state.apply(&env_info, self.engine, &t) { Ok(receipt) => { - flushln!("Transaction executed {:?}", receipt); self.block.archive_set.insert(h.unwrap_or_else(||t.hash())); self.block.archive.push(Entry { transaction: t, receipt: receipt }); Ok(&self.block.archive.last().unwrap().receipt) From 3fe0c3c7897b36fa9ed87ccca275d7c277093264 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 26 Jan 2016 00:26:36 +0100 Subject: [PATCH 7/9] Unignore passingn tests. --- src/tests/chain.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/tests/chain.rs b/src/tests/chain.rs index 89199feed..ba42ae935 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -47,18 +47,18 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } -declare_test!{ignore => BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} // FAILS declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} -declare_test!{ignore => BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcForkUncle, "BlockchainTests/bcForkUncle"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcGasPricerTest, "BlockchainTests/bcGasPricerTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcInvalidRLPTest, "BlockchainTests/bcInvalidRLPTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcRPC_API_Test, "BlockchainTests/bcRPC_API_Test"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} // FAILS (Suicides, GasUsed) -declare_test!{ignore => BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcUncleHeaderValiditiy, "BlockchainTests/bcUncleHeaderValiditiy"}// UNKNOWN -declare_test!{ignore => BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} // UNKNOWN -declare_test!{ignore => BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} // UNKNOWN +declare_test!{ignore => BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} // FAILS +declare_test!{ignore => BlockchainTests_bcForkUncle, "BlockchainTests/bcForkUncle"} // FAILS +declare_test!{BlockchainTests_bcGasPricerTest, "BlockchainTests/bcGasPricerTest"} +declare_test!{BlockchainTests_bcInvalidHeaderTest, "BlockchainTests/bcInvalidHeaderTest"} +declare_test!{ignore => BlockchainTests_bcInvalidRLPTest, "BlockchainTests/bcInvalidRLPTest"} // FAILS +declare_test!{ignore => BlockchainTests_bcMultiChainTest, "BlockchainTests/bcMultiChainTest"} // FAILS +declare_test!{BlockchainTests_bcRPC_API_Test, "BlockchainTests/bcRPC_API_Test"} +declare_test!{ignore => BlockchainTests_bcStateTest, "BlockchainTests/bcStateTest"} // FAILS (Suicides, GasUsed) +declare_test!{BlockchainTests_bcTotalDifficultyTest, "BlockchainTests/bcTotalDifficultyTest"} +declare_test!{ignore => BlockchainTests_bcUncleHeaderValiditiy, "BlockchainTests/bcUncleHeaderValiditiy"} // FAILS +declare_test!{ignore => BlockchainTests_bcUncleTest, "BlockchainTests/bcUncleTest"} // FAILS +declare_test!{ignore => BlockchainTests_bcValidBlockTest, "BlockchainTests/bcValidBlockTest"} // FAILS +declare_test!{ignore => BlockchainTests_bcWalletTest, "BlockchainTests/bcWalletTest"} // FAILS From 2dcfb52b5692d37368e6d60e4d6eb2aad9f2fbbf Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 26 Jan 2016 00:46:24 +0100 Subject: [PATCH 8/9] Closes #213 --- src/tests/chain.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/chain.rs b/src/tests/chain.rs index ba42ae935..d3e978e05 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -47,6 +47,7 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } + declare_test!{ignore => BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} // FAILS declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} declare_test!{ignore => BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} // FAILS From 7065c477a4ec5ac81b3cb24c4b45153f437e9287 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 26 Jan 2016 00:48:01 +0100 Subject: [PATCH 9/9] Closes #70 --- src/tests/chain.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/chain.rs b/src/tests/chain.rs index d3e978e05..ba42ae935 100644 --- a/src/tests/chain.rs +++ b/src/tests/chain.rs @@ -47,7 +47,6 @@ fn do_json_test(json_data: &[u8]) -> Vec { failed } - declare_test!{ignore => BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} // FAILS declare_test!{BlockchainTests_bcForkBlockTest, "BlockchainTests/bcForkBlockTest"} declare_test!{ignore => BlockchainTests_bcForkStressTest, "BlockchainTests/bcForkStressTest"} // FAILS