From c8076b2f9d9ac45e1a431366eaa5710cedfdcccc Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 21 Feb 2016 19:46:29 +0100 Subject: [PATCH 01/33] Threading performance optimizations --- Cargo.lock | 3 +- Cargo.toml | 3 + ethcore/src/block_queue.rs | 146 +++++++++++++++++++----------------- ethcore/src/client.rs | 30 ++++---- ethcore/src/verification.rs | 14 +--- util/sha3/build.rs | 2 +- util/src/lib.rs | 2 + util/src/thread.rs | 43 +++++++++++ 8 files changed, 148 insertions(+), 95 deletions(-) create mode 100644 util/src/thread.rs diff --git a/Cargo.lock b/Cargo.lock index cf747f3cc..50274857f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,7 +151,6 @@ dependencies = [ [[package]] name = "eth-secp256k1" version = "0.5.4" -source = "git+https://github.com/arkpar/rust-secp256k1.git#45503e1de68d909b1862e3f2bdb9e1cdfdff3f1e" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -223,7 +222,7 @@ dependencies = [ "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "eth-secp256k1 0.5.4 (git+https://github.com/arkpar/rust-secp256k1.git)", + "eth-secp256k1 0.5.4", "ethcore-devtools 0.9.99", "heapsize 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 7fdfc2bee..f28829180 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,3 +30,6 @@ travis-nightly = ["ethcore/json-tests", "dev"] [[bin]] path = "parity/main.rs" name = "parity" + +[profile.release] +debug = true diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index c39f158f0..a51a1e900 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -63,7 +63,7 @@ pub struct BlockQueue { panic_handler: Arc, engine: Arc>, more_to_verify: Arc, - verification: Arc>, + verification: Arc, verifiers: Vec>, deleting: Arc, ready_signal: Arc, @@ -98,12 +98,11 @@ impl QueueSignal { } } -#[derive(Default)] struct Verification { - unverified: VecDeque, - verified: VecDeque, - verifying: VecDeque, - bad: HashSet, + unverified: Mutex>, + verified: Mutex>, + verifying: Mutex>, + bad: Mutex>, } const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; @@ -111,7 +110,12 @@ const MAX_UNVERIFIED_QUEUE_SIZE: usize = 50000; impl BlockQueue { /// Creates a new queue instance. pub fn new(engine: Arc>, message_channel: IoChannel) -> BlockQueue { - let verification = Arc::new(Mutex::new(Verification::default())); + let verification = Arc::new(Verification { + unverified: Mutex::new(VecDeque::new()), + verified: Mutex::new(VecDeque::new()), + verifying: Mutex::new(VecDeque::new()), + bad: Mutex::new(HashSet::new()), + }); let more_to_verify = Arc::new(Condvar::new()); let ready_signal = Arc::new(QueueSignal { signalled: AtomicBool::new(false), message_channel: message_channel }); let deleting = Arc::new(AtomicBool::new(false)); @@ -119,7 +123,7 @@ impl BlockQueue { let panic_handler = PanicHandler::new_in_arc(); let mut verifiers: Vec> = Vec::new(); - let thread_count = max(::num_cpus::get(), 3) - 2; + let thread_count = max(::num_cpus::get(), 5) - 0; for i in 0..thread_count { let verification = verification.clone(); let engine = engine.clone(); @@ -133,7 +137,8 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) + lower_thread_priority(); + BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) .expect("Error starting block verification thread") @@ -152,17 +157,17 @@ impl BlockQueue { } } - fn verify(verification: Arc>, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { + fn verify(verification: Arc, engine: Arc>, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { while !deleting.load(AtomicOrdering::Acquire) { { - let mut lock = verification.lock().unwrap(); + let mut unverified = verification.unverified.lock().unwrap(); - if lock.unverified.is_empty() && lock.verifying.is_empty() { + if unverified.is_empty() && verification.verifying.lock().unwrap().is_empty() { empty.notify_all(); } - while lock.unverified.is_empty() && !deleting.load(AtomicOrdering::Acquire) { - lock = wait.wait(lock).unwrap(); + while unverified.is_empty() && !deleting.load(AtomicOrdering::Acquire) { + unverified = wait.wait(unverified).unwrap(); } if deleting.load(AtomicOrdering::Acquire) { @@ -171,39 +176,42 @@ impl BlockQueue { } let block = { - let mut v = verification.lock().unwrap(); - if v.unverified.is_empty() { + let mut unverified = verification.unverified.lock().unwrap(); + if unverified.is_empty() { continue; } - let block = v.unverified.pop_front().unwrap(); - v.verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None }); + let mut verifying = verification.verifying.lock().unwrap(); + let block = unverified.pop_front().unwrap(); + verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None }); block }; let block_hash = block.header.hash(); match verify_block_unordered(block.header, block.bytes, engine.deref().deref()) { Ok(verified) => { - let mut v = verification.lock().unwrap(); - for e in &mut v.verifying { + let mut verifying = verification.verifying.lock().unwrap(); + for e in verifying.iter_mut() { if e.hash == block_hash { e.block = Some(verified); break; } } - if !v.verifying.is_empty() && v.verifying.front().unwrap().hash == block_hash { + if !verifying.is_empty() && verifying.front().unwrap().hash == block_hash { // we're next! - let mut vref = v.deref_mut(); - BlockQueue::drain_verifying(&mut vref.verifying, &mut vref.verified, &mut vref.bad); + let mut verified = verification.verified.lock().unwrap(); + let mut bad = verification.bad.lock().unwrap(); + BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); ready.set(); } }, Err(err) => { - let mut v = verification.lock().unwrap(); + let mut verifying = verification.verifying.lock().unwrap(); + let mut verified = verification.verified.lock().unwrap(); + let mut bad = verification.bad.lock().unwrap(); warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); - v.bad.insert(block_hash.clone()); - v.verifying.retain(|e| e.hash != block_hash); - let mut vref = v.deref_mut(); - BlockQueue::drain_verifying(&mut vref.verifying, &mut vref.verified, &mut vref.bad); + bad.insert(block_hash.clone()); + verifying.retain(|e| e.hash != block_hash); + BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); ready.set(); } } @@ -223,19 +231,21 @@ impl BlockQueue { } /// Clear the queue and stop verification activity. - pub fn clear(&mut self) { - let mut verification = self.verification.lock().unwrap(); - verification.unverified.clear(); - verification.verifying.clear(); - verification.verified.clear(); + pub fn clear(&self) { + let mut unverified = self.verification.unverified.lock().unwrap(); + let mut verifying = self.verification.verifying.lock().unwrap(); + let mut verified = self.verification.verified.lock().unwrap(); + unverified.clear(); + verifying.clear(); + verified.clear(); self.processing.write().unwrap().clear(); } - /// Wait for queue to be empty - pub fn flush(&mut self) { - let mut verification = self.verification.lock().unwrap(); - while !verification.unverified.is_empty() || !verification.verifying.is_empty() { - verification = self.empty.wait(verification).unwrap(); + /// Wait for unverified queue to be empty + pub fn flush(&self) { + let mut unverified = self.verification.unverified.lock().unwrap(); + while !unverified.is_empty() || !self.verification.verifying.lock().unwrap().is_empty() { + unverified = self.empty.wait(unverified).unwrap(); } } @@ -244,27 +254,29 @@ impl BlockQueue { if self.processing.read().unwrap().contains(&hash) { return BlockStatus::Queued; } - if self.verification.lock().unwrap().bad.contains(&hash) { + if self.verification.bad.lock().unwrap().contains(&hash) { return BlockStatus::Bad; } BlockStatus::Unknown } /// Add a block to the queue. - pub fn import_block(&mut self, bytes: Bytes) -> ImportResult { + pub fn import_block(&self, bytes: Bytes) -> ImportResult { let header = BlockView::new(&bytes).header(); let h = header.hash(); - if self.processing.read().unwrap().contains(&h) { - return Err(ImportError::AlreadyQueued); - } { - let mut verification = self.verification.lock().unwrap(); - if verification.bad.contains(&h) { + if self.processing.read().unwrap().contains(&h) { + return Err(ImportError::AlreadyQueued); + } + } + { + let mut bad = self.verification.bad.lock().unwrap(); + if bad.contains(&h) { return Err(ImportError::Bad(None)); } - if verification.bad.contains(&header.parent_hash) { - verification.bad.insert(h.clone()); + if bad.contains(&header.parent_hash) { + bad.insert(h.clone()); return Err(ImportError::Bad(None)); } } @@ -272,39 +284,40 @@ impl BlockQueue { match verify_block_basic(&header, &bytes, self.engine.deref().deref()) { Ok(()) => { self.processing.write().unwrap().insert(h.clone()); - self.verification.lock().unwrap().unverified.push_back(UnVerifiedBlock { header: header, bytes: bytes }); + self.verification.unverified.lock().unwrap().push_back(UnVerifiedBlock { header: header, bytes: bytes }); self.more_to_verify.notify_all(); Ok(h) }, Err(err) => { warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), err); - self.verification.lock().unwrap().bad.insert(h.clone()); + self.verification.bad.lock().unwrap().insert(h.clone()); Err(From::from(err)) } } } /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&mut self, hash: &H256) { - let mut verification_lock = self.verification.lock().unwrap(); - let mut verification = verification_lock.deref_mut(); - verification.bad.insert(hash.clone()); + pub fn mark_as_bad(&self, hash: &H256) { + let mut verified_lock = self.verification.verified.lock().unwrap(); + let mut verified = verified_lock.deref_mut(); + let mut bad = self.verification.bad.lock().unwrap(); + bad.insert(hash.clone()); self.processing.write().unwrap().remove(&hash); let mut new_verified = VecDeque::new(); - for block in verification.verified.drain(..) { - if verification.bad.contains(&block.header.parent_hash) { - verification.bad.insert(block.header.hash()); + for block in verified.drain(..) { + if bad.contains(&block.header.parent_hash) { + bad.insert(block.header.hash()); self.processing.write().unwrap().remove(&block.header.hash()); } else { new_verified.push_back(block); } } - verification.verified = new_verified; + *verified = new_verified; } /// Mark given block as processed - pub fn mark_as_good(&mut self, hashes: &[H256]) { + pub fn mark_as_good(&self, hashes: &[H256]) { let mut processing = self.processing.write().unwrap(); for h in hashes { processing.remove(&h); @@ -312,16 +325,16 @@ impl BlockQueue { } /// Removes up to `max` verified blocks from the queue - pub fn drain(&mut self, max: usize) -> Vec { - let mut verification = self.verification.lock().unwrap(); - let count = min(max, verification.verified.len()); + pub fn drain(&self, max: usize) -> Vec { + let mut verified = self.verification.verified.lock().unwrap(); + let count = min(max, verified.len()); let mut result = Vec::with_capacity(count); for _ in 0..count { - let block = verification.verified.pop_front().unwrap(); + let block = verified.pop_front().unwrap(); result.push(block); } self.ready_signal.reset(); - if !verification.verified.is_empty() { + if !verified.is_empty() { self.ready_signal.set(); } result @@ -329,11 +342,10 @@ impl BlockQueue { /// Get queue status. pub fn queue_info(&self) -> BlockQueueInfo { - let verification = self.verification.lock().unwrap(); BlockQueueInfo { - verified_queue_size: verification.verified.len(), - unverified_queue_size: verification.unverified.len(), - verifying_queue_size: verification.verifying.len(), + unverified_queue_size: self.verification.unverified.lock().unwrap().len(), + verifying_queue_size: self.verification.verifying.lock().unwrap().len(), + verified_queue_size: self.verification.verified.lock().unwrap().len(), } } } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c3ec4b4d0..0c8580117 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -172,7 +172,7 @@ pub struct Client { chain: Arc>, engine: Arc>, state_db: Mutex, - block_queue: RwLock, + block_queue: BlockQueue, report: RwLock, import_lock: Mutex<()>, panic_handler: Arc, @@ -231,7 +231,7 @@ impl Client { chain: chain, engine: engine, state_db: Mutex::new(state_db), - block_queue: RwLock::new(block_queue), + block_queue: block_queue, report: RwLock::new(Default::default()), import_lock: Mutex::new(()), panic_handler: panic_handler @@ -240,7 +240,7 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { - self.block_queue.write().unwrap().flush(); + self.block_queue.flush(); } /// This is triggered by a message coming from a block queue when the block is ready for insertion @@ -248,11 +248,11 @@ impl Client { let mut ret = 0; let mut bad = HashSet::new(); let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.write().unwrap().drain(128); + let blocks = self.block_queue.drain(128); let mut good_blocks = Vec::with_capacity(128); for block in blocks { if bad.contains(&block.header.parent_hash) { - self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); + self.block_queue.mark_as_bad(&block.header.hash()); bad.insert(block.header.hash()); continue; } @@ -260,7 +260,7 @@ impl Client { let header = &block.header; if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + self.block_queue.mark_as_bad(&header.hash()); bad.insert(block.header.hash()); break; }; @@ -268,7 +268,7 @@ impl Client { Some(p) => p, None => { warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + self.block_queue.mark_as_bad(&header.hash()); bad.insert(block.header.hash()); break; }, @@ -292,13 +292,13 @@ impl Client { Err(e) => { warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); bad.insert(block.header.hash()); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + self.block_queue.mark_as_bad(&header.hash()); break; } }; if let Err(e) = verify_block_final(&header, result.block().header()) { warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + self.block_queue.mark_as_bad(&header.hash()); break; } @@ -317,8 +317,8 @@ impl Client { trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); ret += 1; } - self.block_queue.write().unwrap().mark_as_good(&good_blocks); - if !good_blocks.is_empty() && self.block_queue.read().unwrap().queue_info().is_empty() { + self.block_queue.mark_as_good(&good_blocks); + if !good_blocks.is_empty() && self.block_queue.queue_info().is_empty() { io.send(NetworkIoMessage::User(SyncMessage::BlockVerified)).unwrap(); } ret @@ -389,7 +389,7 @@ impl BlockChainClient for Client { let chain = self.chain.read().unwrap(); match Self::block_hash(&chain, id) { Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain, - Some(hash) => self.block_queue.read().unwrap().block_status(&hash), + Some(hash) => self.block_queue.block_status(&hash), None => BlockStatus::Unknown } } @@ -434,15 +434,15 @@ impl BlockChainClient for Client { if self.block_status(BlockId::Hash(header.parent_hash)) == BlockStatus::Unknown { return Err(ImportError::UnknownParent); } - self.block_queue.write().unwrap().import_block(bytes) + self.block_queue.import_block(bytes) } fn queue_info(&self) -> BlockQueueInfo { - self.block_queue.read().unwrap().queue_info() + self.block_queue.queue_info() } fn clear_queue(&self) { - self.block_queue.write().unwrap().clear(); + self.block_queue.clear(); } fn chain_info(&self) -> BlockChainInfo { diff --git a/ethcore/src/verification.rs b/ethcore/src/verification.rs index c7d5e265f..fa9467e95 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification.rs @@ -57,18 +57,12 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res /// Still operates on a individual block /// Returns a PreVerifiedBlock structure populated with transactions pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result { - try!(engine.verify_block_unordered(&header, Some(&bytes))); - for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { - try!(engine.verify_block_unordered(&u, None)); - } // Verify transactions. let mut transactions = Vec::new(); - { - let v = BlockView::new(&bytes); - for t in v.transactions() { - try!(engine.verify_transaction(&t, &header)); - transactions.push(t); - } + let v = BlockView::new(&bytes); + for t in v.transactions() { + try!(engine.verify_transaction(&t, &header)); + transactions.push(t); } Ok(PreVerifiedBlock { header: header, diff --git a/util/sha3/build.rs b/util/sha3/build.rs index bbe16d720..9eb36fdb9 100644 --- a/util/sha3/build.rs +++ b/util/sha3/build.rs @@ -21,6 +21,6 @@ extern crate gcc; fn main() { - gcc::compile_library("libtinykeccak.a", &["src/tinykeccak.c"]); + gcc::Config::new().file("src/tinykeccak.c").flag("-O3").compile("libtinykeccak.a"); } diff --git a/util/src/lib.rs b/util/src/lib.rs index 2b7438cf3..5c8bd4fb0 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -143,6 +143,7 @@ pub mod network; pub mod log; pub mod panics; pub mod keys; +mod thread; pub use common::*; pub use misc::*; @@ -163,4 +164,5 @@ pub use semantic_version::*; pub use network::*; pub use io::*; pub use log::*; +pub use thread::*; diff --git a/util/src/thread.rs b/util/src/thread.rs new file mode 100644 index 000000000..b86ca3e86 --- /dev/null +++ b/util/src/thread.rs @@ -0,0 +1,43 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Thread management helpers + +use libc::{c_int, pthread_self, pthread_t}; + +#[repr(C)] +struct sched_param { + priority: c_int, + padding: c_int, +} + +extern { + fn setpriority(which: c_int, who: c_int, prio: c_int) -> c_int; + fn pthread_setschedparam(thread: pthread_t, policy: c_int, param: *const sched_param) -> c_int; +} +const PRIO_DARWIN_THREAD: c_int = 3; +const PRIO_DARWIN_BG: c_int = 0x1000; +const SCHED_RR: c_int = 2; + +/// Lower thread priority and put it into background mode +#[cfg(target_os="macos")] +pub fn lower_thread_priority() { + let sp = sched_param { priority: 0, padding: 0 }; + if unsafe { pthread_setschedparam(pthread_self(), SCHED_RR, &sp) } == -1 { + trace!("Could not decrease thread piority"); + } + //unsafe { setpriority(PRIO_DARWIN_THREAD, 0, PRIO_DARWIN_BG); } +} From 778fa92ebe82bcf7c739b4750a5443f2ed575802 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 22 Feb 2016 00:36:59 +0100 Subject: [PATCH 02/33] Remove locks from the block chain --- Cargo.lock | 3 +- Cargo.toml | 2 +- ethcore/src/block_queue.rs | 12 +++---- ethcore/src/blockchain.rs | 48 ++++++++++++++++------------ ethcore/src/client.rs | 63 +++++++++++++++++-------------------- ethcore/src/verification.rs | 14 ++++++--- util/src/lib.rs | 2 -- util/src/thread.rs | 43 ------------------------- 8 files changed, 75 insertions(+), 112 deletions(-) delete mode 100644 util/src/thread.rs diff --git a/Cargo.lock b/Cargo.lock index 50274857f..cf747f3cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,6 +151,7 @@ dependencies = [ [[package]] name = "eth-secp256k1" version = "0.5.4" +source = "git+https://github.com/arkpar/rust-secp256k1.git#45503e1de68d909b1862e3f2bdb9e1cdfdff3f1e" dependencies = [ "arrayvec 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -222,7 +223,7 @@ dependencies = [ "crossbeam 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "eth-secp256k1 0.5.4", + "eth-secp256k1 0.5.4 (git+https://github.com/arkpar/rust-secp256k1.git)", "ethcore-devtools 0.9.99", "heapsize 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "igd 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index f28829180..8bc94a3a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,4 +32,4 @@ path = "parity/main.rs" name = "parity" [profile.release] -debug = true +debug = false diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index a51a1e900..ba9867966 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -99,6 +99,7 @@ impl QueueSignal { } struct Verification { + // All locks must be captured in the order declared here. unverified: Mutex>, verified: Mutex>, verifying: Mutex>, @@ -123,7 +124,7 @@ impl BlockQueue { let panic_handler = PanicHandler::new_in_arc(); let mut verifiers: Vec> = Vec::new(); - let thread_count = max(::num_cpus::get(), 5) - 0; + let thread_count = max(::num_cpus::get(), 3) - 2; for i in 0..thread_count { let verification = verification.clone(); let engine = engine.clone(); @@ -137,7 +138,6 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - lower_thread_priority(); BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) @@ -392,7 +392,7 @@ mod tests { #[test] fn can_import_blocks() { - let mut queue = get_test_queue(); + let queue = get_test_queue(); if let Err(e) = queue.import_block(get_good_dummy_block()) { panic!("error importing block that is valid by definition({:?})", e); } @@ -400,7 +400,7 @@ mod tests { #[test] fn returns_error_for_duplicates() { - let mut queue = get_test_queue(); + let queue = get_test_queue(); if let Err(e) = queue.import_block(get_good_dummy_block()) { panic!("error importing block that is valid by definition({:?})", e); } @@ -419,7 +419,7 @@ mod tests { #[test] fn returns_ok_for_drained_duplicates() { - let mut queue = get_test_queue(); + let queue = get_test_queue(); let block = get_good_dummy_block(); let hash = BlockView::new(&block).header().hash().clone(); if let Err(e) = queue.import_block(block) { @@ -436,7 +436,7 @@ mod tests { #[test] fn returns_empty_once_finished() { - let mut queue = get_test_queue(); + let queue = get_test_queue(); queue.import_block(get_good_dummy_block()).expect("error importing block that is valid by definition"); queue.flush(); queue.drain(1); diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain.rs index cc9ff56fd..22d409e8e 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain.rs @@ -16,6 +16,7 @@ //! Blockchain database. +use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrder}; use util::*; use rocksdb::{DB, WriteBatch, Writable}; use header::*; @@ -147,8 +148,9 @@ struct CacheManager { /// /// **Does not do input data verification.** pub struct BlockChain { - pref_cache_size: usize, - max_cache_size: usize, + // All locks must be captured in the order declared here. + pref_cache_size: AtomicUsize, + max_cache_size: AtomicUsize, best_block: RwLock, @@ -166,6 +168,7 @@ pub struct BlockChain { blocks_db: DB, cache_man: RwLock, + insert_lock: Mutex<()> } impl BlockProvider for BlockChain { @@ -261,8 +264,8 @@ impl BlockChain { (0..COLLECTION_QUEUE_SIZE).foreach(|_| cache_man.cache_usage.push_back(HashSet::new())); let bc = BlockChain { - pref_cache_size: 1 << 14, - max_cache_size: 1 << 20, + pref_cache_size: AtomicUsize::new(1 << 14), + max_cache_size: AtomicUsize::new(1 << 20), best_block: RwLock::new(BestBlock::new()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), @@ -273,6 +276,7 @@ impl BlockChain { extras_db: extras_db, blocks_db: blocks_db, cache_man: RwLock::new(cache_man), + insert_lock: Mutex::new(()), }; // load best block @@ -315,9 +319,9 @@ impl BlockChain { } /// Set the cache configuration. - pub fn configure_cache(&mut self, pref_cache_size: usize, max_cache_size: usize) { - self.pref_cache_size = pref_cache_size; - self.max_cache_size = max_cache_size; + pub fn configure_cache(&self, pref_cache_size: usize, max_cache_size: usize) { + self.pref_cache_size.store(pref_cache_size, AtomicOrder::Relaxed); + self.max_cache_size.store(max_cache_size, AtomicOrder::Relaxed); } /// Returns a tree route between `from` and `to`, which is a tuple of: @@ -435,22 +439,26 @@ impl BlockChain { return; } + let _lock = self.insert_lock.lock(); // store block in db self.blocks_db.put(&hash, &bytes).unwrap(); let (batch, new_best, details) = self.block_to_extras_insert_batch(bytes); - // update best block - let mut best_block = self.best_block.write().unwrap(); - if let Some(b) = new_best { - *best_block = b; + { + // update best block + let mut best_block = self.best_block.write().unwrap(); + if let Some(b) = new_best { + *best_block = b; + } } - // update caches - let mut write = self.block_details.write().unwrap(); - write.remove(&header.parent_hash()); - write.insert(hash.clone(), details); - self.note_used(CacheID::Block(hash)); - + { + // update caches + let mut write = self.block_details.write().unwrap(); + write.remove(&header.parent_hash()); + write.insert(hash.clone(), details); + self.note_used(CacheID::Block(hash)); + } // update extras database self.extras_db.write(batch).unwrap(); } @@ -622,17 +630,17 @@ impl BlockChain { /// Ticks our cache system and throws out any old data. pub fn collect_garbage(&self) { - if self.cache_size().total() < self.pref_cache_size { return; } + if self.cache_size().total() < self.pref_cache_size.load(AtomicOrder::Relaxed) { return; } for _ in 0..COLLECTION_QUEUE_SIZE { { - let mut cache_man = self.cache_man.write().unwrap(); let mut blocks = self.blocks.write().unwrap(); let mut block_details = self.block_details.write().unwrap(); let mut block_hashes = self.block_hashes.write().unwrap(); let mut transaction_addresses = self.transaction_addresses.write().unwrap(); let mut block_logs = self.block_logs.write().unwrap(); let mut blocks_blooms = self.blocks_blooms.write().unwrap(); + let mut cache_man = self.cache_man.write().unwrap(); for id in cache_man.cache_usage.pop_back().unwrap().into_iter() { cache_man.in_use.remove(&id); @@ -650,7 +658,7 @@ impl BlockChain { // TODO: handle block_hashes properly. block_hashes.clear(); } - if self.cache_size().total() < self.max_cache_size { break; } + if self.cache_size().total() < self.max_cache_size.load(AtomicOrder::Relaxed) { break; } } // TODO: m_lastCollection = chrono::system_clock::now(); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 0c8580117..68801520c 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -169,7 +169,7 @@ impl ClientReport { /// Blockchain database client backed by a persistent database. Owns and manages a blockchain and a block queue. /// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue. pub struct Client { - chain: Arc>, + chain: Arc, engine: Arc>, state_db: Mutex, block_queue: BlockQueue, @@ -190,7 +190,7 @@ impl Client { dir.push(format!("v{}-sec-pruned", CLIENT_DB_VER_STR)); let path = dir.as_path(); let gb = spec.genesis_block(); - let chain = Arc::new(RwLock::new(BlockChain::new(&gb, path))); + let chain = Arc::new(BlockChain::new(&gb, path)); let mut opts = Options::new(); opts.set_max_open_files(256); opts.create_if_missing(true); @@ -258,13 +258,13 @@ impl Client { } let header = &block.header; - if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { + if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.deref()) { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); self.block_queue.mark_as_bad(&header.hash()); bad.insert(block.header.hash()); break; }; - let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) { + let parent = match self.chain.block_header(&header.parent_hash) { Some(p) => p, None => { warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); @@ -278,7 +278,7 @@ impl Client { last_hashes.resize(256, H256::new()); last_hashes[0] = header.parent_hash.clone(); for i in 0..255 { - match self.chain.read().unwrap().block_details(&last_hashes[i]) { + match self.chain.block_details(&last_hashes[i]) { Some(details) => { last_hashes[i + 1] = details.parent.clone(); }, @@ -304,9 +304,9 @@ impl Client { good_blocks.push(header.hash().clone()); - self.chain.write().unwrap().insert_block(&block.bytes); //TODO: err here? + self.chain.insert_block(&block.bytes); //TODO: err here? let ancient = if header.number() >= HISTORY { Some(header.number() - HISTORY) } else { None }; - match result.drain().commit(header.number(), &header.hash(), ancient.map(|n|(n, self.chain.read().unwrap().block_hash(n).unwrap()))) { + match result.drain().commit(header.number(), &header.hash(), ancient.map(|n|(n, self.chain.block_hash(n).unwrap()))) { Ok(_) => (), Err(e) => { warn!(target: "client", "State DB commit failed: {:?}", e); @@ -331,7 +331,7 @@ impl Client { /// Get info on the cache. pub fn cache_info(&self) -> CacheSize { - self.chain.read().unwrap().cache_size() + self.chain.cache_size() } /// Get the report. @@ -341,12 +341,12 @@ impl Client { /// Tick the client. pub fn tick(&self) { - self.chain.read().unwrap().collect_garbage(); + self.chain.collect_garbage(); } /// Set up the cache behaviour. pub fn configure_cache(&self, pref_cache_size: usize, max_cache_size: usize) { - self.chain.write().unwrap().configure_cache(pref_cache_size, max_cache_size); + self.chain.configure_cache(pref_cache_size, max_cache_size); } fn block_hash(chain: &BlockChain, id: BlockId) -> Option { @@ -361,14 +361,12 @@ impl Client { impl BlockChainClient for Client { fn block_header(&self, id: BlockId) -> Option { - let chain = self.chain.read().unwrap(); - Self::block_hash(&chain, id).and_then(|hash| chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec())) + Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec())) } fn block_body(&self, id: BlockId) -> Option { - let chain = self.chain.read().unwrap(); - Self::block_hash(&chain, id).and_then(|hash| { - chain.block(&hash).map(|bytes| { + Self::block_hash(&self.chain, id).and_then(|hash| { + self.chain.block(&hash).map(|bytes| { let rlp = Rlp::new(&bytes); let mut body = RlpStream::new_list(2); body.append_raw(rlp.at(1).as_raw(), 1); @@ -379,24 +377,21 @@ impl BlockChainClient for Client { } fn block(&self, id: BlockId) -> Option { - let chain = self.chain.read().unwrap(); - Self::block_hash(&chain, id).and_then(|hash| { - chain.block(&hash) + Self::block_hash(&self.chain, id).and_then(|hash| { + self.chain.block(&hash) }) } fn block_status(&self, id: BlockId) -> BlockStatus { - let chain = self.chain.read().unwrap(); - match Self::block_hash(&chain, id) { - Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain, + match Self::block_hash(&self.chain, id) { + Some(ref hash) if self.chain.is_known(hash) => BlockStatus::InChain, Some(hash) => self.block_queue.block_status(&hash), None => BlockStatus::Unknown } } fn block_total_difficulty(&self, id: BlockId) -> Option { - let chain = self.chain.read().unwrap(); - Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) + Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_details(&hash)).map(|d| d.total_difficulty) } fn code(&self, address: &Address) -> Option { @@ -404,18 +399,17 @@ impl BlockChainClient for Client { } fn transaction(&self, id: TransactionId) -> Option { - let chain = self.chain.read().unwrap(); match id { - TransactionId::Hash(ref hash) => chain.transaction_address(hash), - TransactionId::Location(id, index) => Self::block_hash(&chain, id).map(|hash| TransactionAddress { + TransactionId::Hash(ref hash) => self.chain.transaction_address(hash), + TransactionId::Location(id, index) => Self::block_hash(&self.chain, id).map(|hash| TransactionAddress { block_hash: hash, index: index }) - }.and_then(|address| chain.transaction(&address)) + }.and_then(|address| self.chain.transaction(&address)) } fn tree_route(&self, from: &H256, to: &H256) -> Option { - self.chain.read().unwrap().tree_route(from.clone(), to.clone()) + self.chain.tree_route(from.clone(), to.clone()) } fn state_data(&self, _hash: &H256) -> Option { @@ -428,7 +422,7 @@ impl BlockChainClient for Client { fn import_block(&self, bytes: Bytes) -> ImportResult { let header = BlockView::new(&bytes).header(); - if self.chain.read().unwrap().is_known(&header.hash()) { + if self.chain.is_known(&header.hash()) { return Err(ImportError::AlreadyInChain); } if self.block_status(BlockId::Hash(header.parent_hash)) == BlockStatus::Unknown { @@ -446,13 +440,12 @@ impl BlockChainClient for Client { } fn chain_info(&self) -> BlockChainInfo { - let chain = self.chain.read().unwrap(); BlockChainInfo { - total_difficulty: chain.best_block_total_difficulty(), - pending_total_difficulty: chain.best_block_total_difficulty(), - genesis_hash: chain.genesis_hash(), - best_block_hash: chain.best_block_hash(), - best_block_number: From::from(chain.best_block_number()) + total_difficulty: self.chain.best_block_total_difficulty(), + pending_total_difficulty: self.chain.best_block_total_difficulty(), + genesis_hash: self.chain.genesis_hash(), + best_block_hash: self.chain.best_block_hash(), + best_block_number: From::from(self.chain.best_block_number()) } } } diff --git a/ethcore/src/verification.rs b/ethcore/src/verification.rs index fa9467e95..c7d5e265f 100644 --- a/ethcore/src/verification.rs +++ b/ethcore/src/verification.rs @@ -57,12 +57,18 @@ pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Res /// Still operates on a individual block /// Returns a PreVerifiedBlock structure populated with transactions pub fn verify_block_unordered(header: Header, bytes: Bytes, engine: &Engine) -> Result { + try!(engine.verify_block_unordered(&header, Some(&bytes))); + for u in Rlp::new(&bytes).at(2).iter().map(|rlp| rlp.as_val::
()) { + try!(engine.verify_block_unordered(&u, None)); + } // Verify transactions. let mut transactions = Vec::new(); - let v = BlockView::new(&bytes); - for t in v.transactions() { - try!(engine.verify_transaction(&t, &header)); - transactions.push(t); + { + let v = BlockView::new(&bytes); + for t in v.transactions() { + try!(engine.verify_transaction(&t, &header)); + transactions.push(t); + } } Ok(PreVerifiedBlock { header: header, diff --git a/util/src/lib.rs b/util/src/lib.rs index 5c8bd4fb0..2b7438cf3 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -143,7 +143,6 @@ pub mod network; pub mod log; pub mod panics; pub mod keys; -mod thread; pub use common::*; pub use misc::*; @@ -164,5 +163,4 @@ pub use semantic_version::*; pub use network::*; pub use io::*; pub use log::*; -pub use thread::*; diff --git a/util/src/thread.rs b/util/src/thread.rs deleted file mode 100644 index b86ca3e86..000000000 --- a/util/src/thread.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Thread management helpers - -use libc::{c_int, pthread_self, pthread_t}; - -#[repr(C)] -struct sched_param { - priority: c_int, - padding: c_int, -} - -extern { - fn setpriority(which: c_int, who: c_int, prio: c_int) -> c_int; - fn pthread_setschedparam(thread: pthread_t, policy: c_int, param: *const sched_param) -> c_int; -} -const PRIO_DARWIN_THREAD: c_int = 3; -const PRIO_DARWIN_BG: c_int = 0x1000; -const SCHED_RR: c_int = 2; - -/// Lower thread priority and put it into background mode -#[cfg(target_os="macos")] -pub fn lower_thread_priority() { - let sp = sched_param { priority: 0, padding: 0 }; - if unsafe { pthread_setschedparam(pthread_self(), SCHED_RR, &sp) } == -1 { - trace!("Could not decrease thread piority"); - } - //unsafe { setpriority(PRIO_DARWIN_THREAD, 0, PRIO_DARWIN_BG); } -} From cb4d17825bc6353461549a4a9c03424cda525cae Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 29 Feb 2016 19:49:29 +0100 Subject: [PATCH 03/33] Fixed lock order --- Cargo.toml | 1 + ethcore/src/blockchain/blockchain.rs | 78 +++++++++++++++------------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 25b7caa85..df3beef49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,3 +35,4 @@ name = "parity" [profile.release] debug = false +lto = false diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 7339f3a1a..f30a674e6 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -433,48 +433,56 @@ impl BlockChain { batch.put(b"best", &update.info.hash).unwrap(); // These cached values must be updated atomically - let mut best_block = self.best_block.write().unwrap(); - let mut write_hashes = self.block_hashes.write().unwrap(); - let mut write_txs = self.transaction_addresses.write().unwrap(); + { + let mut best_block = self.best_block.write().unwrap(); + let mut write_hashes = self.block_hashes.write().unwrap(); + let mut write_txs = self.transaction_addresses.write().unwrap(); - // update best block - match update.info.location { - BlockLocation::Branch => (), - _ => { - *best_block = BestBlock { - hash: update.info.hash, - number: update.info.number, - total_difficulty: update.info.total_difficulty - }; + // update best block + match update.info.location { + BlockLocation::Branch => (), + _ => { + *best_block = BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty + }; + } + } + + for (number, hash) in &update.block_hashes { + batch.put_extras(number, hash); + write_hashes.remove(number); + } + + for (hash, tx_address) in &update.transactions_addresses { + batch.put_extras(hash, tx_address); + write_txs.remove(hash); } } - for (number, hash) in &update.block_hashes { - batch.put_extras(number, hash); - write_hashes.remove(number); + { + let mut write_details = self.block_details.write().unwrap(); + for (hash, details) in update.block_details.into_iter() { + batch.put_extras(&hash, &details); + write_details.insert(hash, details); + } } - let mut write_details = self.block_details.write().unwrap(); - for (hash, details) in update.block_details.into_iter() { - batch.put_extras(&hash, &details); - write_details.insert(hash, details); + { + let mut write_receipts = self.block_receipts.write().unwrap(); + for (hash, receipt) in &update.block_receipts { + batch.put_extras(hash, receipt); + write_receipts.remove(hash); + } } - let mut write_receipts = self.block_receipts.write().unwrap(); - for (hash, receipt) in &update.block_receipts { - batch.put_extras(hash, receipt); - write_receipts.remove(hash); - } - - for (hash, tx_address) in &update.transactions_addresses { - batch.put_extras(hash, tx_address); - write_txs.remove(hash); - } - - let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for (bloom_hash, blocks_bloom) in &update.blocks_blooms { - batch.put_extras(bloom_hash, blocks_bloom); - write_blocks_blooms.remove(bloom_hash); + { + let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); + for (bloom_hash, blocks_bloom) in &update.blocks_blooms { + batch.put_extras(bloom_hash, blocks_bloom); + write_blocks_blooms.remove(bloom_hash); + } } // update extras database @@ -580,7 +588,7 @@ impl BlockChain { /// This function returns modified transaction addresses. fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { let block = BlockView::new(block_bytes); - let transaction_hashes = block.transaction_hashes(); + let transaction_hashes = block.transaction_hashes(); transaction_hashes.into_iter() .enumerate() From d0129ff67b5e69436df88a6758e81d880e924a23 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 29 Feb 2016 21:15:39 +0100 Subject: [PATCH 04/33] Fixed cache memory leak --- ethcore/src/blockchain/blockchain.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index f30a674e6..23e9aaac9 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -465,6 +465,7 @@ impl BlockChain { let mut write_details = self.block_details.write().unwrap(); for (hash, details) in update.block_details.into_iter() { batch.put_extras(&hash, &details); + self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash.clone())); write_details.insert(hash, details); } } @@ -769,6 +770,14 @@ impl BlockChain { // TODO: handle block_hashes properly. block_hashes.clear(); + + blocks.shrink_to_fit(); + block_details.shrink_to_fit(); + block_hashes.shrink_to_fit(); + transaction_addresses.shrink_to_fit(); + block_logs.shrink_to_fit(); + blocks_blooms.shrink_to_fit(); + block_receipts.shrink_to_fit(); } if self.cache_size().total() < self.max_cache_size.load(AtomicOrder::Relaxed) { break; } } From c889d9b3eb21c1b54989d741c6b34bed4bfc30ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 25 Feb 2016 16:58:18 +0100 Subject: [PATCH 05/33] Exposing transaction queue pending in RPC --- rpc/src/v1/impls/eth.rs | 13 ++++++++++++- rpc/src/v1/traits/eth.rs | 13 ++++++++----- sync/src/chain.rs | 3 +++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 00bce5437..6d66a2c6d 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -152,7 +152,7 @@ impl Eth for EthClient { } } - fn block_transaction_count(&self, params: Params) -> Result { + fn block_transaction_count_by_hash(&self, params: Params) -> Result { from_params::<(H256,)>(params) .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).transactions_count()), @@ -160,6 +160,17 @@ impl Eth for EthClient { }) } + fn block_transaction_count_by_number(&self, params: Params) -> Result { + from_params::<(BlockNumber,)>(params) + .and_then(|(block_number,)| match block_number { + BlockNumber::Pending => to_value(&take_weak!(self.sync).status().transaction_queue_pending), + _ => match take_weak!(self.client).block(block_number.into()) { + Some(bytes) => to_value(&BlockView::new(&bytes).transactions_count()), + None => Ok(Value::Null) + } + }) + } + fn block_uncles_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { diff --git a/rpc/src/v1/traits/eth.rs b/rpc/src/v1/traits/eth.rs index d2aeb0f9e..8c24dd38c 100644 --- a/rpc/src/v1/traits/eth.rs +++ b/rpc/src/v1/traits/eth.rs @@ -55,12 +55,15 @@ pub trait Eth: Sized + Send + Sync + 'static { /// Returns block with given number. fn block_by_number(&self, _: Params) -> Result { rpc_unimplemented!() } - + /// Returns the number of transactions sent from given address at given time (block number). fn transaction_count(&self, _: Params) -> Result { rpc_unimplemented!() } - /// Returns the number of transactions in a block. - fn block_transaction_count(&self, _: Params) -> Result { rpc_unimplemented!() } + /// Returns the number of transactions in a block given block hash. + fn block_transaction_count_by_hash(&self, _: Params) -> Result { rpc_unimplemented!() } + + /// Returns the number of transactions in a block given block number. + fn block_transaction_count_by_number(&self, _: Params) -> Result { rpc_unimplemented!() } /// Returns the number of uncles in a given block. fn block_uncles_count(&self, _: Params) -> Result { rpc_unimplemented!() } @@ -130,8 +133,8 @@ pub trait Eth: Sized + Send + Sync + 'static { delegate.add_method("eth_balance", Eth::balance); delegate.add_method("eth_getStorageAt", Eth::storage_at); delegate.add_method("eth_getTransactionCount", Eth::transaction_count); - delegate.add_method("eth_getBlockTransactionCountByHash", Eth::block_transaction_count); - delegate.add_method("eth_getBlockTransactionCountByNumber", Eth::block_transaction_count); + delegate.add_method("eth_getBlockTransactionCountByHash", Eth::block_transaction_count_by_hash); + delegate.add_method("eth_getBlockTransactionCountByNumber", Eth::block_transaction_count_by_number); delegate.add_method("eth_getUncleCountByBlockHash", Eth::block_uncles_count); delegate.add_method("eth_getUncleCountByBlockNumber", Eth::block_uncles_count); delegate.add_method("eth_code", Eth::code_at); diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 9edab791a..90f7b0d2a 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -140,6 +140,8 @@ pub struct SyncStatus { pub num_active_peers: usize, /// Heap memory used in bytes pub mem_used: usize, + /// Number of pending transactions in queue + pub transaction_queue_pending: usize, } #[derive(PartialEq, Eq, Debug, Clone)] @@ -255,6 +257,7 @@ impl ChainSync { blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.len(), num_active_peers: self.peers.values().filter(|p| p.asking != PeerAsking::Nothing).count(), + transaction_queue_pending: self.transaction_queue.lock().unwrap().status().pending, mem_used: // TODO: https://github.com/servo/heapsize/pull/50 // self.downloading_hashes.heap_size_of_children() From 324e070581305ded5b9c40942f86837756f2116d Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 2 Mar 2016 01:24:06 +0100 Subject: [PATCH 06/33] Reverted some changes --- ethcore/src/block_queue.rs | 3 +-- util/sha3/build.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index f61eb565d..de411c6e2 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -302,8 +302,7 @@ impl BlockQueue { if self.processing.read().unwrap().contains(&h) { return Err(ImportError::AlreadyQueued); } - } - { + let mut bad = self.verification.bad.lock().unwrap(); if bad.contains(&h) { return Err(ImportError::Bad(None)); diff --git a/util/sha3/build.rs b/util/sha3/build.rs index 9eb36fdb9..bbe16d720 100644 --- a/util/sha3/build.rs +++ b/util/sha3/build.rs @@ -21,6 +21,6 @@ extern crate gcc; fn main() { - gcc::Config::new().file("src/tinykeccak.c").flag("-O3").compile("libtinykeccak.a"); + gcc::compile_library("libtinykeccak.a", &["src/tinykeccak.c"]); } From af5ed8b5f7af5b576fed3181dc03c45fdee50f1b Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 4 Mar 2016 20:10:07 +0300 Subject: [PATCH 07/33] rpc-signing-extend --- parity/main.rs | 1 + rpc/src/v1/impls/eth.rs | 17 ++++++++++++++++- rpc/src/v1/impls/personal.rs | 21 ++++++++++----------- rpc/src/v1/types/bytes.rs | 32 ++++++++++++++++++++++++++++++-- rpc/src/v1/types/mod.rs.in | 2 ++ rpc/src/v1/types/transaction.rs | 13 +++++++++++++ 6 files changed, 72 insertions(+), 14 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index b991f36cd..91a884beb 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -157,6 +157,7 @@ fn setup_rpc_server(client: Arc, sync: Arc, url: &str, cors_dom server.add_delegate(EthClient::new(&client, &sync).to_delegate()); server.add_delegate(EthFilterClient::new(&client).to_delegate()); server.add_delegate(NetClient::new(&sync).to_delegate()); + server.add_delegate(PersonalClient::new(&client).to_delegate()); server.start_async(url, cors_domain); } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 2313d5114..16b68f90f 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -29,7 +29,7 @@ use ethcore::views::*; use ethcore::ethereum::Ethash; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; -use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, OptionalValue, Index, Filter, Log}; +use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, TransactionRequest, OptionalValue, Index, Filter, Log}; use v1::helpers::{PollFilter, PollManager}; /// Eth rpc implementation. @@ -253,6 +253,21 @@ impl Eth for EthClient { to_value(&true) }) } + + fn send_transaction(&self, params: Params) -> Result { + from_params::<(TransactionRequest, )>(params) + .and_then(|(transaction_request, )| { + let client = take_weak!(self.client); + let store = client.secret_store().read().unwrap(); + match store.account_secret(&transaction_request.from) { + Ok(_) => { + // todo: actually sign and push to queue transaction here + Ok(Value::Bool(true)) + }, + Err(_) => { Ok(Value::Bool(false ))} + } + }) + } } /// Eth filter rpc implementation. diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index 48e1b1c6a..a2788b9d9 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -18,28 +18,27 @@ use std::sync::{Arc, Weak}; use jsonrpc_core::*; use v1::traits::Personal; -use util::keys::store::*; use util::Address; -use std::sync::RwLock; +use ethcore::client::Client; /// Account management (personal) rpc implementation. pub struct PersonalClient { - secret_store: Weak>, + client: Weak, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(store: &Arc>) -> Self { + pub fn new(client: &Arc) -> Self { PersonalClient { - secret_store: Arc::downgrade(store), + client: Arc::downgrade(client), } } } impl Personal for PersonalClient { fn accounts(&self, _: Params) -> Result { - let store_wk = take_weak!(self.secret_store); - let store = store_wk.read().unwrap(); + let client = take_weak!(self.client); + let store = client.secret_store().read().unwrap(); match store.accounts() { Ok(account_list) => { Ok(Value::Array(account_list.iter() @@ -54,8 +53,8 @@ impl Personal for PersonalClient { fn new_account(&self, params: Params) -> Result { from_params::<(String, )>(params).and_then( |(pass, )| { - let store_wk = take_weak!(self.secret_store); - let mut store = store_wk.write().unwrap(); + let client = take_weak!(self.client); + let mut store = client.secret_store().write().unwrap(); match store.new_account(&pass) { Ok(address) => Ok(Value::String(format!("{:?}", address))), Err(_) => Err(Error::internal_error()) @@ -67,8 +66,8 @@ impl Personal for PersonalClient { fn unlock_account(&self, params: Params) -> Result { from_params::<(Address, String, u64)>(params).and_then( |(account, account_pass, _)|{ - let store_wk = take_weak!(self.secret_store); - let store = store_wk.read().unwrap(); + let client = take_weak!(self.client); + let store = client.secret_store().read().unwrap(); match store.unlock_account(&account, &account_pass) { Ok(_) => Ok(Value::Bool(true)), Err(_) => Ok(Value::Bool(false)), diff --git a/rpc/src/v1/types/bytes.rs b/rpc/src/v1/types/bytes.rs index f09f24e4d..44809ac70 100644 --- a/rpc/src/v1/types/bytes.rs +++ b/rpc/src/v1/types/bytes.rs @@ -15,7 +15,9 @@ // along with Parity. If not, see . use rustc_serialize::hex::ToHex; -use serde::{Serialize, Serializer}; +use serde::{Serialize, Serializer, Deserialize, Deserializer, Error}; +use serde::de::Visitor; +use util::common::FromHex; /// Wrapper structure around vector of bytes. #[derive(Debug)] @@ -36,7 +38,7 @@ impl Default for Bytes { } impl Serialize for Bytes { - fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: Serializer { let mut serialized = "0x".to_owned(); serialized.push_str(self.0.to_hex().as_ref()); @@ -44,6 +46,32 @@ impl Serialize for Bytes { } } +impl Deserialize for Bytes { + fn deserialize(deserializer: &mut D) -> Result + where D: Deserializer { + deserializer.deserialize(BytesVisitor) + } +} + +struct BytesVisitor; + +impl Visitor for BytesVisitor { + type Value = Bytes; + + fn visit_str(&mut self, value: &str) -> Result where E: Error { + if value.len() >= 2 && &value[0..2] == "0x" { + Ok(Bytes::new(FromHex::from_hex(&value[2..]).unwrap_or_else(|_| vec![]))) + } else { + Err(Error::custom("invalid hex")) + } + } + + fn visit_string(&mut self, value: String) -> Result where E: Error { + self.visit_str(value.as_ref()) + } +} + + #[cfg(test)] mod tests { use super::*; diff --git a/rpc/src/v1/types/mod.rs.in b/rpc/src/v1/types/mod.rs.in index 34c1f1cff..2b2390ecb 100644 --- a/rpc/src/v1/types/mod.rs.in +++ b/rpc/src/v1/types/mod.rs.in @@ -33,3 +33,5 @@ pub use self::log::Log; pub use self::optionals::OptionalValue; pub use self::sync::{SyncStatus, SyncInfo}; pub use self::transaction::Transaction; +pub use self::transaction::TransactionRequest; + diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index 232cf0bf3..7d40d8a49 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -17,6 +17,7 @@ use util::numbers::*; use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; +use serde::{Deserializer, Error}; #[derive(Debug, Default, Serialize)] pub struct Transaction { @@ -37,6 +38,18 @@ pub struct Transaction { pub input: Bytes } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct TransactionRequest { + pub from: Address, + pub to: Option
, + #[serde(rename="gasPrice")] + pub gas_price: Option, + pub gas: Option, + pub value: Option, + pub data: Bytes, + pub nonce: Option, +} + impl From for Transaction { fn from(t: LocalizedTransaction) -> Transaction { Transaction { From 1aaae7b55333625c572fc77cc722ea98c2517825 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 16:42:02 +0300 Subject: [PATCH 08/33] [ci skip] codegen bug --- rpc/src/v1/types/transaction.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index 7d40d8a49..c24bcd08f 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -18,6 +18,8 @@ use util::numbers::*; use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; use serde::{Deserializer, Error}; +use ethcore; +use util; #[derive(Debug, Default, Serialize)] pub struct Transaction { @@ -50,6 +52,22 @@ pub struct TransactionRequest { pub nonce: Option, } +impl TransactionRequest { + fn to_eth(self) -> (ethcore::transaction::Transaction, Address) { + (ethcore::transaction::Transaction { + nonce: self.nonce.unwrap_or(U256::zero()), + action: match self.to { + None => ethcore::transaction::Action::Create, + Some(addr) => ethcore::transaction::Action::Call(addr) + }, + gas: self.gas.unwrap_or(U256::zero()), + gas_price: self.gas_price.unwrap_or(U256::zero()), + value: self.value.unwrap_or(U256::zero()), + data: { let (ref x) = self.data; x } + }, self.from) + } +} + impl From for Transaction { fn from(t: LocalizedTransaction) -> Transaction { Transaction { From bb8a79f18c44575d59ce68d4a9b5c03009679585 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 18:29:01 +0300 Subject: [PATCH 09/33] finalizing --- rpc/src/v1/impls/eth.rs | 12 ++++++++---- rpc/src/v1/types/bytes.rs | 1 + rpc/src/v1/types/transaction.rs | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 16b68f90f..91ccaa05a 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -260,11 +260,15 @@ impl Eth for EthClient { let client = take_weak!(self.client); let store = client.secret_store().read().unwrap(); match store.account_secret(&transaction_request.from) { - Ok(_) => { - // todo: actually sign and push to queue transaction here - Ok(Value::Bool(true)) + Ok(secret) => { + let sync = take_weak!(self.sync); + let (transaction, _) = transaction_request.to_eth(); + let signed_transaction = transaction.sign(&secret); + let hash = signed_transaction.hash(); + sync.insert_transaction(signed_transaction); + to_value(&hash) }, - Err(_) => { Ok(Value::Bool(false ))} + Err(_) => { to_value(&U256::zero()) } } }) } diff --git a/rpc/src/v1/types/bytes.rs b/rpc/src/v1/types/bytes.rs index 44809ac70..466fbebde 100644 --- a/rpc/src/v1/types/bytes.rs +++ b/rpc/src/v1/types/bytes.rs @@ -28,6 +28,7 @@ impl Bytes { pub fn new(bytes: Vec) -> Bytes { Bytes(bytes) } + pub fn to_vec(self) -> Vec { let Bytes(x) = self; x } } impl Default for Bytes { diff --git a/rpc/src/v1/types/transaction.rs b/rpc/src/v1/types/transaction.rs index c24bcd08f..17b42cfcf 100644 --- a/rpc/src/v1/types/transaction.rs +++ b/rpc/src/v1/types/transaction.rs @@ -19,7 +19,6 @@ use ethcore::transaction::{LocalizedTransaction, Action}; use v1::types::{Bytes, OptionalValue}; use serde::{Deserializer, Error}; use ethcore; -use util; #[derive(Debug, Default, Serialize)] pub struct Transaction { @@ -53,7 +52,8 @@ pub struct TransactionRequest { } impl TransactionRequest { - fn to_eth(self) -> (ethcore::transaction::Transaction, Address) { + /// maps transaction request to the transaction that can be signed and inserted + pub fn to_eth(self) -> (ethcore::transaction::Transaction, Address) { (ethcore::transaction::Transaction { nonce: self.nonce.unwrap_or(U256::zero()), action: match self.to { @@ -63,7 +63,7 @@ impl TransactionRequest { gas: self.gas.unwrap_or(U256::zero()), gas_price: self.gas_price.unwrap_or(U256::zero()), value: self.value.unwrap_or(U256::zero()), - data: { let (ref x) = self.data; x } + data: self.data.to_vec() }, self.from) } } From ae1c1b918faea16a81d2fe582446a148fdea3c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sat, 5 Mar 2016 16:51:01 +0100 Subject: [PATCH 10/33] Fixing compilation --- sync/src/transaction_queue.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 8b38c64ad..24bb772d7 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -297,12 +297,6 @@ impl TransactionQueue { } } self.future.enforce_limit(&mut self.by_hash); - - // And now lets check if there is some chain of transactions in future - // that should be placed in current - if let Some(new_current_top) = self.move_future_txs(sender.clone(), current_nonce, current_nonce) { - self.last_nonces.insert(sender, new_current_top); - } } From e100ecbeacf6bac923f4b8e416621f98668b4830 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sat, 5 Mar 2016 23:47:28 +0300 Subject: [PATCH 11/33] exposing in lib --- sync/src/chain.rs | 4 ++++ sync/src/lib.rs | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index ddf30854a..fd1771045 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1294,6 +1294,10 @@ impl ChainSync { transaction_queue.add_all(txs, |a| chain.nonce(a)); }); } + + pub fn transaction_queue(&self) -> &Mutex { + return &self.transaction_queue; + } } #[cfg(test)] diff --git a/sync/src/lib.rs b/sync/src/lib.rs index d67a09f3b..a6480b0ad 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -128,6 +128,16 @@ impl EthSync { pub fn restart(&mut self, io: &mut NetworkContext) { self.sync.write().unwrap().restart(&mut NetSyncIo::new(io, self.chain.deref())); } + + /// Insert transaction in transaction queue + pub fn insert_transaction(&self, transaction: SignedTransaction) { + use util::numbers::*; + + let nonce_fn = |a: &Address| self.chain.state().nonce(a) + U256::one(); + let sync = self.sync.write().unwrap(); + let mut queue = sync.transaction_queue().lock().unwrap(); + queue.add(transaction, &nonce_fn); + } } impl NetworkProtocolHandler for EthSync { From ad8135668392aa733dea014231a3a56469dae5fc Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sun, 6 Mar 2016 00:48:00 +0300 Subject: [PATCH 12/33] fix namespace --- sync/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/src/lib.rs b/sync/src/lib.rs index a6480b0ad..e352144bd 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -130,7 +130,7 @@ impl EthSync { } /// Insert transaction in transaction queue - pub fn insert_transaction(&self, transaction: SignedTransaction) { + pub fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction) { use util::numbers::*; let nonce_fn = |a: &Address| self.chain.state().nonce(a) + U256::one(); From 655bb0ed5db4eb014d959b03b3edc43cfb5ebf4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 8 Mar 2016 12:36:06 +0100 Subject: [PATCH 13/33] Additional documentation for transaction queue --- sync/src/lib.rs | 1 + sync/src/transaction_queue.rs | 103 +++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/sync/src/lib.rs b/sync/src/lib.rs index b5869642c..39a06af8f 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -72,6 +72,7 @@ mod chain; mod io; mod range_collection; mod transaction_queue; +pub use transaction_queue::TransactionQueue; #[cfg(test)] mod tests; diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 3e0d931b5..8270c6e27 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -17,6 +17,67 @@ // TODO [todr] - own transactions should have higher priority //! Transaction Queue +//! +//! TransactionQueue keeps track of all transactions seen by the node (received from other peers) and own transactions +//! and orders them by priority. Top priority transactions are those with low nonce height (difference between +//! transaction's nonce and next nonce expected from this sender). If nonces are equal transaction's gas price is used +//! for comparison (higher gas price = higher priority). +//! +//! # Usage Example +//! +//! ```rust +//! extern crate ethcore_util as util; +//! extern crate ethcore; +//! extern crate ethsync; +//! extern crate rustc_serialize; +//! +//! use util::crypto::KeyPair; +//! use util::hash::Address; +//! use util::numbers::{Uint, U256}; +//! use ethsync::TransactionQueue; +//! use ethcore::transaction::*; +//! use rustc_serialize::hex::FromHex; +//! +//! fn main() { +//! let key = KeyPair::create().unwrap(); +//! let t1 = Transaction { action: Action::Create, value: U256::from(100), data: "3331600055".from_hex().unwrap(), +//! gas: U256::from(100_000), gas_price: U256::one(), nonce: U256::from(10) }; +//! let t2 = Transaction { action: Action::Create, value: U256::from(100), data: "3331600055".from_hex().unwrap(), +//! gas: U256::from(100_000), gas_price: U256::one(), nonce: U256::from(11) }; +//! +//! let st1 = t1.sign(&key.secret()); +//! let st2 = t2.sign(&key.secret()); +//! let default_nonce = |_a: &Address| U256::from(10); +//! +//! let mut txq = TransactionQueue::new(); +//! txq.add(st2.clone(), &default_nonce); +//! txq.add(st1.clone(), &default_nonce); +//! +//! // Check status +//! assert_eq!(txq.status().pending, 2); +//! // Check top transactions +//! let top = txq.top_transactions(3); +//! assert_eq!(top.len(), 2); +//! assert_eq!(top[0], st1); +//! assert_eq!(top[1], st2); +//! +//! // And when transaction is removed (but nonce haven't changed) +//! // it will move invalid transactions to future +//! txq.remove(&st1.hash(), &default_nonce); +//! assert_eq!(txq.status().pending, 0); +//! assert_eq!(txq.status().future, 1); +//! assert_eq!(txq.top_transactions(3).len(), 0); +//! } +//! +//! +//! # Maintaing valid state +//! +//! 1. Whenever transaction is imported to queue (to queue) all other transactions from this sender are revalidated in current. It means that they are moved to future and back again (height recalculation & gap filling). +//! 2. Whenever transaction is removed: +//! - When it's removed from `future` - all `future` transactions heights are recalculated and then +//! we check if the transactions should go to `current` (comparing state nonce) +//! - When it's removed from `current` - all transactions from this sender (`current` & `future`) are recalculated. +//! use std::cmp::{Ordering}; use std::collections::{HashMap, BTreeSet}; @@ -27,9 +88,16 @@ use ethcore::transaction::*; #[derive(Clone, Debug)] +/// Light structure used to identify transaction and it's order struct TransactionOrder { + /// Primary ordering factory. Difference between transaction nonce and expected nonce in state + /// (e.g. Tx(nonce:5), State(nonce:0) -> height: 5) + /// High nonce_height = Low priority (processed later) nonce_height: U256, + /// Gas Price of the transaction. + /// Low gas price = Low priority (processed later) gas_price: U256, + /// Hash to identify associated transaction hash: H256, } @@ -70,7 +138,7 @@ impl Ord for TransactionOrder { let a_gas = self.gas_price; let b_gas = b.gas_price; if a_gas != b_gas { - return a_gas.cmp(&b_gas); + return b_gas.cmp(&a_gas); } // Compare hashes @@ -78,6 +146,7 @@ impl Ord for TransactionOrder { } } +/// Verified transaction (with sender) struct VerifiedTransaction { transaction: SignedTransaction } @@ -101,6 +170,11 @@ impl VerifiedTransaction { } } +/// Holds transactions accessible by (address, nonce) and by priority +/// +/// TransactionSet keeps number of entries below limit, but it doesn't +/// automatically happen during `insert/remove` operations. +/// You have to call `enforce_limit` to remove lowest priority transactions from set. struct TransactionSet { by_priority: BTreeSet, by_address: Table, @@ -108,11 +182,15 @@ struct TransactionSet { } impl TransactionSet { + /// Inserts `TransactionOrder` to this set fn insert(&mut self, sender: Address, nonce: U256, order: TransactionOrder) -> Option { self.by_priority.insert(order.clone()); self.by_address.insert(sender, nonce, order) } + /// Remove low priority transactions if there is more then specified by given `limit`. + /// + /// It drops transactions from this set but also removes associated `VerifiedTransaction`. fn enforce_limit(&mut self, by_hash: &mut HashMap) { let len = self.by_priority.len(); if len <= self.limit { @@ -134,6 +212,7 @@ impl TransactionSet { } } + /// Drop transaction from this set (remove from `by_priority` and `by_address`) fn drop(&mut self, sender: &Address, nonce: &U256) -> Option { if let Some(tx_order) = self.by_address.remove(sender, nonce) { self.by_priority.remove(&tx_order); @@ -142,6 +221,7 @@ impl TransactionSet { None } + /// Drop all transactions. fn clear(&mut self) { self.by_priority.clear(); self.by_address.clear(); @@ -260,6 +340,8 @@ impl TransactionQueue { // We will either move transaction to future or remove it completely // so there will be no transactions from this sender in current self.last_nonces.remove(&sender); + // First update height of transactions in future to avoid collisions + self.update_future(&sender, current_nonce); // This should move all current transactions to future and remove old transactions self.move_all_to_future(&sender, current_nonce); // And now lets check if there is some chain of transactions in future @@ -269,6 +351,7 @@ impl TransactionQueue { } } + /// Update height of all transactions in future transactions set. fn update_future(&mut self, sender: &Address, current_nonce: U256) { // We need to drain all transactions for current sender from future and reinsert them with updated height let all_nonces_from_sender = match self.future.by_address.row(&sender) { @@ -281,6 +364,8 @@ impl TransactionQueue { } } + /// Drop all transactions from given sender from `current`. + /// Either moves them to `future` or removes them from queue completely. fn move_all_to_future(&mut self, sender: &Address, current_nonce: U256) { let all_nonces_from_sender = match self.current.by_address.row(&sender) { Some(row_map) => row_map.keys().cloned().collect::>(), @@ -300,7 +385,7 @@ impl TransactionQueue { } - /// Returns top transactions from the queue + /// Returns top transactions from the queue ordered by priority. pub fn top_transactions(&self, size: usize) -> Vec { self.current.by_priority .iter() @@ -318,6 +403,8 @@ impl TransactionQueue { self.last_nonces.clear(); } + /// Checks if there are any transactions in `future` that should actually be promoted to `current` + /// (because nonce matches). fn move_matching_future_to_current(&mut self, address: Address, mut current_nonce: U256, first_nonce: U256) { { let by_nonce = self.future.by_address.row_mut(&address); @@ -339,6 +426,14 @@ impl TransactionQueue { self.last_nonces.insert(address, current_nonce - U256::one()); } + /// Adds VerifiedTransaction to this queue. + /// + /// Determines if it should be placed in current or future. When transaction is + /// imported to `current` also checks if there are any `future` transactions that should be promoted because of + /// this. + /// + /// It ignores transactions that has already been imported (same `hash`) and replaces the transaction + /// iff `(address, nonce)` is the same but `gas_price` is higher. fn import_tx(&mut self, tx: VerifiedTransaction, fetch_nonce: &T) where T: Fn(&Address) -> U256 { @@ -377,6 +472,10 @@ impl TransactionQueue { self.current.enforce_limit(&mut self.by_hash); } + /// Replaces transaction in given set (could be `future` or `current`). + /// + /// If there is already transaction with same `(sender, nonce)` it will be replaced iff `gas_price` is higher. + /// One of the transactions is dropped from set and also removed from queue entirely (from `by_hash`). fn replace_transaction(tx: VerifiedTransaction, base_nonce: U256, set: &mut TransactionSet, by_hash: &mut HashMap) { let order = TransactionOrder::for_transaction(&tx, base_nonce); let hash = tx.hash(); From 799d3bd2c8e15e71a027caf7e8c836b578163161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 8 Mar 2016 12:42:32 +0100 Subject: [PATCH 14/33] Fixing doc test for queue --- sync/src/transaction_queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 8270c6e27..4b4a6226b 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -68,7 +68,7 @@ //! assert_eq!(txq.status().future, 1); //! assert_eq!(txq.top_transactions(3).len(), 0); //! } -//! +//! ``` //! //! # Maintaing valid state //! From 082a4d9078cff6b90f4b01ca3614cc9c3825f265 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Mar 2016 17:31:43 +0100 Subject: [PATCH 15/33] jsonrpc uses client and sync interfaces as a preparetion for jsonrpc tests --- ethcore/src/client.rs | 73 +++++++++++++++++++++------------------ parity/main.rs | 2 +- rpc/src/v1/impls/eth.rs | 25 +++++++------- rpc/src/v1/impls/net.rs | 12 +++---- sync/src/lib.rs | 18 +++++++--- sync/src/tests/helpers.rs | 9 +++++ 6 files changed, 81 insertions(+), 58 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 8471666aa..374011f71 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -182,6 +182,13 @@ pub trait BlockChainClient : Sync + Send { /// Returns logs matching given filter. fn logs(&self, filter: Filter) -> Vec; + + /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. + fn sealing_block(&self) -> &Mutex>; + + /// Submit `seal` as a valid solution for the header of `pow_hash`. + /// Will check the seal, but not actually insert the block into the chain. + fn submit_seal(&self, pow_hash: H256, seal: Vec) -> Result<(), Error>; } #[derive(Default, Clone, Debug, Eq, PartialEq)] @@ -511,39 +518,6 @@ impl Client where V: Verifier { trace!("Sealing: number={}, hash={}, diff={}", b.hash(), b.block().header().difficulty(), b.block().header().number()); *self.sealing_block.lock().unwrap() = Some(b); } - - /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. - pub fn sealing_block(&self) -> &Mutex> { - if self.sealing_block.lock().unwrap().is_none() { - self.sealing_enabled.store(true, atomic::Ordering::Relaxed); - // TODO: Above should be on a timer that resets after two blocks have arrived without being asked for. - self.prepare_sealing(); - } - &self.sealing_block - } - - /// Submit `seal` as a valid solution for the header of `pow_hash`. - /// Will check the seal, but not actually insert the block into the chain. - pub fn submit_seal(&self, pow_hash: H256, seal: Vec) -> Result<(), Error> { - let mut maybe_b = self.sealing_block.lock().unwrap(); - match *maybe_b { - Some(ref b) if b.hash() == pow_hash => {} - _ => { return Err(Error::PowHashInvalid); } - } - - let b = maybe_b.take(); - match b.unwrap().try_seal(self.engine.deref().deref(), seal) { - Err(old) => { - *maybe_b = Some(old); - Err(Error::PowInvalid) - } - Ok(sealed) => { - // TODO: commit DB from `sealed.drain` and make a VerifiedBlock to skip running the transactions twice. - try!(self.import_block(sealed.rlp_bytes())); - Ok(()) - } - } - } } // TODO: need MinerService MinerIoHandler @@ -702,6 +676,39 @@ impl BlockChainClient for Client where V: Verifier { }) .collect() } + + /// Grab the `ClosedBlock` that we want to be sealed. Comes as a mutex that you have to lock. + fn sealing_block(&self) -> &Mutex> { + if self.sealing_block.lock().unwrap().is_none() { + self.sealing_enabled.store(true, atomic::Ordering::Relaxed); + // TODO: Above should be on a timer that resets after two blocks have arrived without being asked for. + self.prepare_sealing(); + } + &self.sealing_block + } + + /// Submit `seal` as a valid solution for the header of `pow_hash`. + /// Will check the seal, but not actually insert the block into the chain. + fn submit_seal(&self, pow_hash: H256, seal: Vec) -> Result<(), Error> { + let mut maybe_b = self.sealing_block.lock().unwrap(); + match *maybe_b { + Some(ref b) if b.hash() == pow_hash => {} + _ => { return Err(Error::PowHashInvalid); } + } + + let b = maybe_b.take(); + match b.unwrap().try_seal(self.engine.deref().deref(), seal) { + Err(old) => { + *maybe_b = Some(old); + Err(Error::PowInvalid) + } + Ok(sealed) => { + // TODO: commit DB from `sealed.drain` and make a VerifiedBlock to skip running the transactions twice. + try!(self.import_block(sealed.rlp_bytes())); + Ok(()) + } + } + } } impl MayPanic for Client { diff --git a/parity/main.rs b/parity/main.rs index ceb58e31e..1cd2970b4 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -48,7 +48,7 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethsync::{EthSync, SyncConfig}; +use ethsync::{EthSync, SyncConfig, SyncStatusProvider}; use docopt::Docopt; use daemonize::Daemonize; use number_prefix::{binary_prefix, Standalone, Prefixed}; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 7113c55b1..abcb54ab7 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -17,7 +17,7 @@ //! Eth rpc implementation. use std::collections::HashMap; use std::sync::{Arc, Weak, Mutex, RwLock}; -use ethsync::{EthSync, SyncState}; +use ethsync::{SyncStatusProvider, SyncState}; use jsonrpc_core::*; use util::numbers::*; use util::sha3::*; @@ -25,7 +25,6 @@ use util::rlp::encode; use ethcore::client::*; use ethcore::block::{IsBlock}; use ethcore::views::*; -//#[macro_use] extern crate log; use ethcore::ethereum::Ethash; use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; @@ -33,15 +32,15 @@ use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncIn use v1::helpers::{PollFilter, PollManager}; /// Eth rpc implementation. -pub struct EthClient { - client: Weak, - sync: Weak, +pub struct EthClient where C: BlockChainClient, S: SyncStatusProvider { + client: Weak, + sync: Weak, hashrates: RwLock>, } -impl EthClient { +impl EthClient where C: BlockChainClient, S: SyncStatusProvider { /// Creates new EthClient. - pub fn new(client: &Arc, sync: &Arc) -> Self { + pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { client: Arc::downgrade(client), sync: Arc::downgrade(sync), @@ -95,7 +94,7 @@ impl EthClient { } } -impl Eth for EthClient { +impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncStatusProvider + 'static { fn protocol_version(&self, params: Params) -> Result { match params { Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), @@ -256,14 +255,14 @@ impl Eth for EthClient { } /// Eth filter rpc implementation. -pub struct EthFilterClient { - client: Weak, +pub struct EthFilterClient where C: BlockChainClient { + client: Weak, polls: Mutex>, } -impl EthFilterClient { +impl EthFilterClient where C: BlockChainClient { /// Creates new Eth filter client. - pub fn new(client: &Arc) -> Self { + pub fn new(client: &Arc) -> Self { EthFilterClient { client: Arc::downgrade(client), polls: Mutex::new(PollManager::new()) @@ -271,7 +270,7 @@ impl EthFilterClient { } } -impl EthFilter for EthFilterClient { +impl EthFilter for EthFilterClient where C: BlockChainClient + 'static { fn new_filter(&self, params: Params) -> Result { from_params::<(Filter,)>(params) .and_then(|(filter,)| { diff --git a/rpc/src/v1/impls/net.rs b/rpc/src/v1/impls/net.rs index 9e24caad2..a686ed66f 100644 --- a/rpc/src/v1/impls/net.rs +++ b/rpc/src/v1/impls/net.rs @@ -17,24 +17,24 @@ //! Net rpc implementation. use std::sync::{Arc, Weak}; use jsonrpc_core::*; -use ethsync::EthSync; +use ethsync::SyncStatusProvider; use v1::traits::Net; /// Net rpc implementation. -pub struct NetClient { - sync: Weak +pub struct NetClient where S: SyncStatusProvider { + sync: Weak } -impl NetClient { +impl NetClient where S: SyncStatusProvider { /// Creates new NetClient. - pub fn new(sync: &Arc) -> Self { + pub fn new(sync: &Arc) -> Self { NetClient { sync: Arc::downgrade(sync) } } } -impl Net for NetClient { +impl Net for NetClient where S: SyncStatusProvider + 'static { fn version(&self, _: Params) -> Result { Ok(Value::U64(take_weak!(self.sync).status().protocol_version as u64)) } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 74541660d..427a58e15 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -93,6 +93,12 @@ impl Default for SyncConfig { } } +/// Current sync status +pub trait SyncStatusProvider: Send + Sync { + /// Get sync status + fn status(&self) -> SyncStatus; +} + /// Ethereum network protocol handler pub struct EthSync { /// Shared blockchain client. TODO: this should evetually become an IPC endpoint @@ -114,11 +120,6 @@ impl EthSync { sync } - /// Get sync status - pub fn status(&self) -> SyncStatus { - self.sync.read().unwrap().status() - } - /// Stop sync pub fn stop(&mut self, io: &mut NetworkContext) { self.sync.write().unwrap().abort(&mut NetSyncIo::new(io, self.chain.deref())); @@ -130,6 +131,13 @@ impl EthSync { } } +impl SyncStatusProvider for EthSync { + /// Get sync status + fn status(&self) -> SyncStatus { + self.sync.read().unwrap().status() + } +} + impl NetworkProtocolHandler for EthSync { fn initialize(&self, io: &NetworkContext) { io.register_timer(0, 1000).expect("Error registering sync timer"); diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index e170a4a85..e7d5cf57f 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -25,6 +25,7 @@ use ethcore::receipt::Receipt; use ethcore::transaction::LocalizedTransaction; use ethcore::filter::Filter; use ethcore::log_entry::LocalizedLogEntry; +use ethcore::block::ClosedBlock; pub struct TestBlockChainClient { pub blocks: RwLock>, @@ -125,6 +126,14 @@ impl BlockChainClient for TestBlockChainClient { unimplemented!(); } + fn sealing_block(&self) -> &Mutex> { + unimplemented!(); + } + + fn submit_seal(&self, _pow_hash: H256, _seal: Vec) -> Result<(), Error> { + unimplemented!(); + } + fn block_header(&self, id: BlockId) -> Option { self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).map(|r| Rlp::new(r).at(0).as_raw().to_vec())) } From 8b042ac875f0abe9968f70d76b44773cd64c4350 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Mar 2016 21:55:23 +0100 Subject: [PATCH 16/33] blockchain import_route --- ethcore/src/blockchain/block_info.rs | 6 +- ethcore/src/blockchain/blockchain.rs | 55 +++++++++--- ethcore/src/blockchain/import_route.rs | 119 +++++++++++++++++++++++++ ethcore/src/blockchain/mod.rs | 2 + 4 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 ethcore/src/blockchain/import_route.rs diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs index ce639bfed..335bdbb4e 100644 --- a/ethcore/src/blockchain/block_info.rs +++ b/ethcore/src/blockchain/block_info.rs @@ -18,6 +18,7 @@ use util::numbers::{U256,H256}; use header::BlockNumber; /// Brief info about inserted block. +#[derive(Clone)] pub struct BlockInfo { /// Block hash. pub hash: H256, @@ -30,6 +31,7 @@ pub struct BlockInfo { } /// Describes location of newly inserted block. +#[derive(Clone)] pub enum BlockLocation { /// It's part of the canon chain. CanonChain, @@ -42,6 +44,8 @@ pub enum BlockLocation { /// Hash of the newest common ancestor with old canon chain. ancestor: H256, /// Hashes of the blocks between ancestor and this block. - route: Vec + route: Vec, + /// Hashes of the blocks which were invalidated. + old_route: Vec, } } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e529f50af..e57f7208a 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -28,7 +28,7 @@ use blockchain::best_block::BestBlock; use blockchain::bloom_indexer::BloomIndexer; use blockchain::tree_route::TreeRoute; use blockchain::update::ExtrasUpdate; -use blockchain::CacheSize; +use blockchain::{CacheSize, ImportRoute}; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; @@ -414,14 +414,14 @@ impl BlockChain { /// Inserts the block into backing cache database. /// Expects the block to be valid and already verified. /// If the block is already known, does nothing. - pub fn insert_block(&self, bytes: &[u8], receipts: Vec) { + pub fn insert_block(&self, bytes: &[u8], receipts: Vec) -> ImportRoute { // create views onto rlp let block = BlockView::new(bytes); let header = block.header_view(); let hash = header.sha3(); if self.is_known(&hash) { - return; + return ImportRoute::none(); } // store block in db @@ -435,8 +435,10 @@ impl BlockChain { block_receipts: self.prepare_block_receipts_update(receipts, &info), transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), blocks_blooms: self.prepare_block_blooms_update(bytes, &info), - info: info + info: info.clone(), }); + + ImportRoute::from(info) } /// Applies extras update. @@ -549,9 +551,14 @@ impl BlockChain { match route.blocks.len() { 0 => BlockLocation::CanonChain, - _ => BlockLocation::BranchBecomingCanonChain { - ancestor: route.ancestor, - route: route.blocks.into_iter().skip(route.index).collect() + _ => { + let old_route = route.blocks.iter().take(route.index).cloned().collect::>(); + + BlockLocation::BranchBecomingCanonChain { + ancestor: route.ancestor, + route: route.blocks.into_iter().skip(route.index).collect(), + old_route: old_route.into_iter().rev().collect(), + } } } } else { @@ -572,7 +579,7 @@ impl BlockChain { BlockLocation::CanonChain => { block_hashes.insert(number, info.hash.clone()); }, - BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route, .. } => { let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; @@ -661,7 +668,7 @@ impl BlockChain { ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) .add_bloom(&header.log_bloom(), header.number() as usize) }, - BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route, .. } => { let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; @@ -825,7 +832,7 @@ mod tests { use rustc_serialize::hex::FromHex; use util::hash::*; use util::sha3::Hashable; - use blockchain::{BlockProvider, BlockChain, BlockChainConfig}; + use blockchain::{BlockProvider, BlockChain, BlockChainConfig, ImportRoute}; use tests::helpers::*; use devtools::*; use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer}; @@ -943,10 +950,30 @@ mod tests { let temp = RandomTempPath::new(); let bc = BlockChain::new(BlockChainConfig::default(), &genesis, temp.as_path()); - bc.insert_block(&b1, vec![]); - bc.insert_block(&b2, vec![]); - bc.insert_block(&b3a, vec![]); - bc.insert_block(&b3b, vec![]); + let ir1 = bc.insert_block(&b1, vec![]); + let ir2 = bc.insert_block(&b2, vec![]); + let ir3b = bc.insert_block(&b3b, vec![]); + let ir3a = bc.insert_block(&b3a, vec![]); + + assert_eq!(ir1, ImportRoute { + validated_blocks: vec![b1_hash], + invalidated_blocks: vec![], + }); + + assert_eq!(ir2, ImportRoute { + validated_blocks: vec![b2_hash], + invalidated_blocks: vec![], + }); + + assert_eq!(ir3b, ImportRoute { + validated_blocks: vec![b3b_hash], + invalidated_blocks: vec![], + }); + + assert_eq!(ir3a, ImportRoute { + validated_blocks: vec![b3a_hash], + invalidated_blocks: vec![b3b_hash], + }); assert_eq!(bc.best_block_hash(), best_block_hash); assert_eq!(bc.block_number(&genesis_hash).unwrap(), 0); diff --git a/ethcore/src/blockchain/import_route.rs b/ethcore/src/blockchain/import_route.rs new file mode 100644 index 000000000..10629f2cb --- /dev/null +++ b/ethcore/src/blockchain/import_route.rs @@ -0,0 +1,119 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Import route. + +use util::hash::H256; +use blockchain::block_info::{BlockInfo, BlockLocation}; + +/// Import route for newly inserted block. +#[derive(Debug, PartialEq)] +pub struct ImportRoute { + /// Blocks that were invalidated by new block. + pub invalidated_blocks: Vec, + /// Blocks that were validted by new block. + pub validated_blocks: Vec, +} + +impl ImportRoute { + pub fn none() -> Self { + ImportRoute { + invalidated_blocks: vec![], + validated_blocks: vec![], + } + } +} + +impl From for ImportRoute { + fn from(info: BlockInfo) -> ImportRoute { + match info.location { + BlockLocation::CanonChain => ImportRoute { + invalidated_blocks: vec![], + validated_blocks: vec![info.hash], + }, + BlockLocation::Branch => ImportRoute::none(), + BlockLocation::BranchBecomingCanonChain { mut route, old_route, .. } => { + route.push(info.hash); + ImportRoute { + invalidated_blocks: old_route, + validated_blocks: route, + } + } + } + } +} + +#[cfg(test)] +mod tests { + use util::hash::H256; + use util::numbers::U256; + use blockchain::block_info::{BlockInfo, BlockLocation}; + use blockchain::ImportRoute; + + #[test] + fn import_route_none() { + assert_eq!(ImportRoute::none(), ImportRoute { + validated_blocks: vec![], + invalidated_blocks: vec![], + }); + } + + #[test] + fn import_route_branch() { + let info = BlockInfo { + hash: H256::from(U256::from(1)), + number: 0, + total_difficulty: U256::from(0), + location: BlockLocation::Branch, + }; + + assert_eq!(ImportRoute::from(info), ImportRoute::none()); + } + + #[test] + fn import_route_canon_chain() { + let info = BlockInfo { + hash: H256::from(U256::from(1)), + number: 0, + total_difficulty: U256::from(0), + location: BlockLocation::CanonChain, + }; + + assert_eq!(ImportRoute::from(info), ImportRoute { + invalidated_blocks: vec![], + validated_blocks: vec![H256::from(U256::from(1))], + }); + } + + #[test] + fn import_route_branch_becoming_canon_chain() { + let info = BlockInfo { + hash: H256::from(U256::from(2)), + number: 0, + total_difficulty: U256::from(0), + location: BlockLocation::BranchBecomingCanonChain { + ancestor: H256::from(U256::from(0)), + route: vec![H256::from(U256::from(1))], + old_route: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], + } + }; + + assert_eq!(ImportRoute::from(info), ImportRoute { + invalidated_blocks: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], + validated_blocks: vec![H256::from(U256::from(1)), H256::from(U256::from(2))], + }); + } +} diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs index b0679b563..6559d8364 100644 --- a/ethcore/src/blockchain/mod.rs +++ b/ethcore/src/blockchain/mod.rs @@ -25,7 +25,9 @@ mod tree_route; mod update; #[cfg(test)] mod generator; +mod import_route; pub use self::blockchain::{BlockProvider, BlockChain, BlockChainConfig}; pub use self::cache::CacheSize; pub use self::tree_route::TreeRoute; +pub use self::import_route::ImportRoute; From f397fb210f3ff0695595a28d3dc5483f59b07848 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Mar 2016 00:11:35 +0100 Subject: [PATCH 17/33] fixed typo --- ethcore/src/blockchain/import_route.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/import_route.rs b/ethcore/src/blockchain/import_route.rs index 10629f2cb..b0d76ef6e 100644 --- a/ethcore/src/blockchain/import_route.rs +++ b/ethcore/src/blockchain/import_route.rs @@ -24,7 +24,7 @@ use blockchain::block_info::{BlockInfo, BlockLocation}; pub struct ImportRoute { /// Blocks that were invalidated by new block. pub invalidated_blocks: Vec, - /// Blocks that were validted by new block. + /// Blocks that were validated by new block. pub validated_blocks: Vec, } From 0a7cda09ffc9ec876599b23f2955763ab0ae6539 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Mar 2016 10:17:17 +0100 Subject: [PATCH 18/33] changed route name to enacted and retracted --- ethcore/src/blockchain/block_info.rs | 4 +-- ethcore/src/blockchain/blockchain.rs | 30 ++++++++++----------- ethcore/src/blockchain/import_route.rs | 36 +++++++++++++------------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs index 335bdbb4e..cf16a8834 100644 --- a/ethcore/src/blockchain/block_info.rs +++ b/ethcore/src/blockchain/block_info.rs @@ -44,8 +44,8 @@ pub enum BlockLocation { /// Hash of the newest common ancestor with old canon chain. ancestor: H256, /// Hashes of the blocks between ancestor and this block. - route: Vec, + enacted: Vec, /// Hashes of the blocks which were invalidated. - old_route: Vec, + retracted: Vec, } } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e57f7208a..d67c1b7f1 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -552,12 +552,12 @@ impl BlockChain { match route.blocks.len() { 0 => BlockLocation::CanonChain, _ => { - let old_route = route.blocks.iter().take(route.index).cloned().collect::>(); + let retracted = route.blocks.iter().take(route.index).cloned().collect::>(); BlockLocation::BranchBecomingCanonChain { ancestor: route.ancestor, - route: route.blocks.into_iter().skip(route.index).collect(), - old_route: old_route.into_iter().rev().collect(), + enacted: route.blocks.into_iter().skip(route.index).collect(), + retracted: retracted.into_iter().rev().collect(), } } } @@ -579,11 +579,11 @@ impl BlockChain { BlockLocation::CanonChain => { block_hashes.insert(number, info.hash.clone()); }, - BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route, .. } => { + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref enacted, .. } => { let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - for (index, hash) in route.iter().cloned().enumerate() { + for (index, hash) in enacted.iter().cloned().enumerate() { block_hashes.insert(start_number + index as BlockNumber, hash); } @@ -668,11 +668,11 @@ impl BlockChain { ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) .add_bloom(&header.log_bloom(), header.number() as usize) }, - BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route, .. } => { + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref enacted, .. } => { let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - let mut blooms: Vec = route.iter() + let mut blooms: Vec = enacted.iter() .map(|hash| self.block(hash).unwrap()) .map(|bytes| BlockView::new(&bytes).header_view().log_bloom()) .collect(); @@ -956,23 +956,23 @@ mod tests { let ir3a = bc.insert_block(&b3a, vec![]); assert_eq!(ir1, ImportRoute { - validated_blocks: vec![b1_hash], - invalidated_blocks: vec![], + enacted: vec![b1_hash], + retracted: vec![], }); assert_eq!(ir2, ImportRoute { - validated_blocks: vec![b2_hash], - invalidated_blocks: vec![], + enacted: vec![b2_hash], + retracted: vec![], }); assert_eq!(ir3b, ImportRoute { - validated_blocks: vec![b3b_hash], - invalidated_blocks: vec![], + enacted: vec![b3b_hash], + retracted: vec![], }); assert_eq!(ir3a, ImportRoute { - validated_blocks: vec![b3a_hash], - invalidated_blocks: vec![b3b_hash], + enacted: vec![b3a_hash], + retracted: vec![b3b_hash], }); assert_eq!(bc.best_block_hash(), best_block_hash); diff --git a/ethcore/src/blockchain/import_route.rs b/ethcore/src/blockchain/import_route.rs index b0d76ef6e..262b70899 100644 --- a/ethcore/src/blockchain/import_route.rs +++ b/ethcore/src/blockchain/import_route.rs @@ -23,16 +23,16 @@ use blockchain::block_info::{BlockInfo, BlockLocation}; #[derive(Debug, PartialEq)] pub struct ImportRoute { /// Blocks that were invalidated by new block. - pub invalidated_blocks: Vec, + pub retracted: Vec, /// Blocks that were validated by new block. - pub validated_blocks: Vec, + pub enacted: Vec, } impl ImportRoute { pub fn none() -> Self { ImportRoute { - invalidated_blocks: vec![], - validated_blocks: vec![], + retracted: vec![], + enacted: vec![], } } } @@ -41,15 +41,15 @@ impl From for ImportRoute { fn from(info: BlockInfo) -> ImportRoute { match info.location { BlockLocation::CanonChain => ImportRoute { - invalidated_blocks: vec![], - validated_blocks: vec![info.hash], + retracted: vec![], + enacted: vec![info.hash], }, BlockLocation::Branch => ImportRoute::none(), - BlockLocation::BranchBecomingCanonChain { mut route, old_route, .. } => { - route.push(info.hash); + BlockLocation::BranchBecomingCanonChain { mut enacted, retracted, .. } => { + enacted.push(info.hash); ImportRoute { - invalidated_blocks: old_route, - validated_blocks: route, + retracted: retracted, + enacted: enacted, } } } @@ -66,8 +66,8 @@ mod tests { #[test] fn import_route_none() { assert_eq!(ImportRoute::none(), ImportRoute { - validated_blocks: vec![], - invalidated_blocks: vec![], + enacted: vec![], + retracted: vec![], }); } @@ -93,8 +93,8 @@ mod tests { }; assert_eq!(ImportRoute::from(info), ImportRoute { - invalidated_blocks: vec![], - validated_blocks: vec![H256::from(U256::from(1))], + retracted: vec![], + enacted: vec![H256::from(U256::from(1))], }); } @@ -106,14 +106,14 @@ mod tests { total_difficulty: U256::from(0), location: BlockLocation::BranchBecomingCanonChain { ancestor: H256::from(U256::from(0)), - route: vec![H256::from(U256::from(1))], - old_route: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], + enacted: vec![H256::from(U256::from(1))], + retracted: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], } }; assert_eq!(ImportRoute::from(info), ImportRoute { - invalidated_blocks: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], - validated_blocks: vec![H256::from(U256::from(1)), H256::from(U256::from(2))], + retracted: vec![H256::from(U256::from(3)), H256::from(U256::from(4))], + enacted: vec![H256::from(U256::from(1)), H256::from(U256::from(2))], }); } } From a2046b429f9120a0532dc1da3fc4467fa20c6469 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 10 Mar 2016 13:27:33 +0400 Subject: [PATCH 19/33] exposing only one func --- sync/src/chain.rs | 9 +++++++-- sync/src/lib.rs | 3 +-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 7294570fe..ea9a47da2 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -43,6 +43,7 @@ use io::SyncIo; use transaction_queue::TransactionQueue; use time; use super::SyncConfig; +use ethcore; known_heap_size!(0, PeerInfo, Header, HeaderId); @@ -1300,8 +1301,12 @@ impl ChainSync { // TODO [todr] propagate transactions? } - pub fn transaction_queue(&self) -> &Mutex { - return &self.transaction_queue; + /// Add transaction to the transaction queue + pub fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction, fetch_nonce: &T) + where T: Fn(&Address) -> U256 + { + let mut queue = self.transaction_queue.lock().unwrap(); + queue.add(transaction, fetch_nonce); } } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index fdcf79749..c9eb792a1 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -135,8 +135,7 @@ impl EthSync { let nonce_fn = |a: &Address| self.chain.state().nonce(a) + U256::one(); let sync = self.sync.write().unwrap(); - let mut queue = sync.transaction_queue().lock().unwrap(); - queue.add(transaction, &nonce_fn); + sync.insert_transaction(transaction, &nonce_fn); } } From ff12b53ba69899dd4a700ff58720bdf6d29ca22c Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 10 Mar 2016 10:40:16 +0100 Subject: [PATCH 20/33] Stop workers before stopping event loop --- util/src/io/service.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/util/src/io/service.rs b/util/src/io/service.rs index 83fa71b8a..8a34ee80a 100644 --- a/util/src/io/service.rs +++ b/util/src/io/service.rs @@ -153,7 +153,7 @@ struct UserTimer { pub struct IoManager where Message: Send + Sync { timers: Arc>>, handlers: Vec>>, - _workers: Vec, + workers: Vec, worker_channel: chase_lev::Worker>, work_ready: Arc, } @@ -180,7 +180,7 @@ impl IoManager where Message: Send + Sync + Clone + 'static { timers: Arc::new(RwLock::new(HashMap::new())), handlers: Vec::new(), worker_channel: worker, - _workers: workers, + workers: workers, work_ready: work_ready, }; try!(event_loop.run(&mut io)); @@ -230,7 +230,10 @@ impl Handler for IoManager where Message: Send + Clone + Sync fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { match msg { - IoMessage::Shutdown => event_loop.shutdown(), + IoMessage::Shutdown => { + self.workers.clear(); + event_loop.shutdown(); + }, IoMessage::AddHandler { handler } => { let handler_id = { self.handlers.push(handler.clone()); From 276768a82600045a2f95767d5d5c6dbb72873753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 10 Mar 2016 11:11:59 +0100 Subject: [PATCH 21/33] Failing test case for #656 --- sync/src/transaction_queue.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 3e0d931b5..38e70d1fc 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -541,6 +541,28 @@ mod test { assert_eq!(top[0], tx); } + #[test] + fn should_correctly_update_futures_when_removing() { + // given + let prev_nonce = |a: &Address| default_nonce(a) - U256::one(); + let next2_nonce = |a: &Address| default_nonce(a) + U256::from(2); + + let mut txq = TransactionQueue::new(); + + let (tx, tx2) = new_txs(U256::from(1)); + txq.add(tx.clone(), &prev_nonce); + txq.add(tx2.clone(), &prev_nonce); + assert_eq!(txq.status().future, 2); + + // when + txq.remove(&tx.hash(), &next2_nonce); + // should remove both transactions since they are not valid + + // then + assert_eq!(txq.status().pending, 0); + assert_eq!(txq.status().future, 0); + } + #[test] fn should_move_transactions_if_gap_filled() { // given From 0cf405527e80879a6e97fae68707b86ec67403be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 10 Mar 2016 11:14:25 +0100 Subject: [PATCH 22/33] Fixing update height bug --- sync/src/transaction_queue.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sync/src/transaction_queue.rs b/sync/src/transaction_queue.rs index 38e70d1fc..ac8debfc9 100644 --- a/sync/src/transaction_queue.rs +++ b/sync/src/transaction_queue.rs @@ -277,7 +277,12 @@ impl TransactionQueue { }; for k in all_nonces_from_sender { let order = self.future.drop(&sender, &k).unwrap(); - self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); + if k >= current_nonce { + self.future.insert(sender.clone(), k, order.update_height(k, current_nonce)); + } else { + // Remove the transaction completely + self.by_hash.remove(&order.hash); + } } } From 6d0578e19c5b5442ccdb42d695fb6a70238cf6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 10 Mar 2016 11:16:54 +0100 Subject: [PATCH 23/33] Additional explanation for ordering of commit/insert_block --- ethcore/src/client.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 874fc9646..aaf5fd728 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -396,7 +396,8 @@ impl Client where V: Verifier { .commit(header.number(), &header.hash(), ancient) .expect("State DB commit failed."); - // And update the chain + // And update the chain after commit to prevent race conditions + // (when something is in chain but you are not able to fetch details) self.chain.write().unwrap() .insert_block(&block.bytes, receipts); From 6681aaf76af521a4c67bfa4d85fbaedaa4029915 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Mar 2016 11:32:10 +0100 Subject: [PATCH 24/33] split client into few submodules --- ethcore/src/{ => client}/client.rs | 123 ++--------------------------- ethcore/src/client/config.rs | 31 ++++++++ ethcore/src/client/ids.rs | 44 +++++++++++ ethcore/src/client/mod.rs | 102 ++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 117 deletions(-) rename ethcore/src/{ => client}/client.rs (85%) create mode 100644 ethcore/src/client/config.rs create mode 100644 ethcore/src/client/ids.rs create mode 100644 ethcore/src/client/mod.rs diff --git a/ethcore/src/client.rs b/ethcore/src/client/client.rs similarity index 85% rename from ethcore/src/client.rs rename to ethcore/src/client/client.rs index b342cef15..2f9536b2e 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client/client.rs @@ -20,7 +20,6 @@ use std::marker::PhantomData; use std::sync::atomic::AtomicBool; use util::*; use util::panics::*; -use blockchain::{BlockChain, BlockProvider}; use views::BlockView; use error::*; use header::{BlockNumber}; @@ -28,7 +27,6 @@ use state::State; use spec::Spec; use engine::Engine; use views::HeaderView; -use block_queue::BlockQueue; use service::{NetSyncMessage, SyncMessage}; use env_info::LastHashes; use verification::*; @@ -38,32 +36,9 @@ use extras::TransactionAddress; use filter::Filter; use log_entry::LocalizedLogEntry; use util::keys::store::SecretStore; -pub use block_queue::{BlockQueueConfig, BlockQueueInfo}; -pub use blockchain::{TreeRoute, BlockChainConfig, CacheSize as BlockChainCacheSize}; - -/// Uniquely identifies block. -#[derive(Debug, PartialEq, Clone)] -pub enum BlockId { - /// Block's sha3. - /// Querying by hash is always faster. - Hash(H256), - /// Block number within canon blockchain. - Number(BlockNumber), - /// Earliest block (genesis). - Earliest, - /// Latest mined block. - Latest -} - -/// Uniquely identifies transaction. -#[derive(Debug, PartialEq, Clone)] -pub enum TransactionId { - /// Transaction's sha3. - Hash(H256), - /// Block id and transaction index within this block. - /// Querying by block position is always faster. - Location(BlockId, usize) -} +use block_queue::{BlockQueue, BlockQueueInfo}; +use blockchain::{BlockChain, BlockProvider, TreeRoute, CacheSize as BlockChainCacheSize}; +use client::{BlockId, TransactionId, ClientConfig, BlockChainClient}; /// General block status #[derive(Debug, Eq, PartialEq)] @@ -78,30 +53,6 @@ pub enum BlockStatus { Unknown, } -/// Client configuration. Includes configs for all sub-systems. -#[derive(Debug)] -pub struct ClientConfig { - /// Block queue configuration. - pub queue: BlockQueueConfig, - /// Blockchain configuration. - pub blockchain: BlockChainConfig, - /// Prefer journal rather than archive. - pub prefer_journal: bool, - /// The name of the client instance. - pub name: String, -} - -impl Default for ClientConfig { - fn default() -> ClientConfig { - ClientConfig { - queue: Default::default(), - blockchain: Default::default(), - prefer_journal: false, - name: Default::default(), - } - } -} - /// Information about the blockchain gathered together. #[derive(Debug)] pub struct BlockChainInfo { @@ -123,72 +74,8 @@ impl fmt::Display for BlockChainInfo { } } -/// Blockchain database client. Owns and manages a blockchain and a block queue. -pub trait BlockChainClient : Sync + Send { - /// Get raw block header data by block id. - fn block_header(&self, id: BlockId) -> Option; - - /// Get raw block body data by block id. - /// Block body is an RLP list of two items: uncles and transactions. - fn block_body(&self, id: BlockId) -> Option; - - /// Get raw block data by block header hash. - fn block(&self, id: BlockId) -> Option; - - /// Get block status by block header hash. - fn block_status(&self, id: BlockId) -> BlockStatus; - - /// Get block total difficulty. - fn block_total_difficulty(&self, id: BlockId) -> Option; - - /// Get address nonce. - fn nonce(&self, address: &Address) -> U256; - - /// Get block hash. - fn block_hash(&self, id: BlockId) -> Option; - - /// Get address code. - fn code(&self, address: &Address) -> Option; - - /// Get transaction with given hash. - fn transaction(&self, id: TransactionId) -> Option; - - /// Get a tree route between `from` and `to`. - /// See `BlockChain::tree_route`. - fn tree_route(&self, from: &H256, to: &H256) -> Option; - - /// Get latest state node - fn state_data(&self, hash: &H256) -> Option; - - /// Get raw block receipts data by block header hash. - fn block_receipts(&self, hash: &H256) -> Option; - - /// Import a block into the blockchain. - fn import_block(&self, bytes: Bytes) -> ImportResult; - - /// Get block queue information. - fn queue_info(&self) -> BlockQueueInfo; - - /// Clear block queue and abort all import activity. - fn clear_queue(&self); - - /// Get blockchain information. - fn chain_info(&self) -> BlockChainInfo; - - /// Get the best block header. - fn best_block_header(&self) -> Bytes { - self.block_header(BlockId::Hash(self.chain_info().best_block_hash)).unwrap() - } - - /// Returns numbers of blocks containing given bloom. - fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockId, to_block: BlockId) -> Option>; - - /// Returns logs matching given filter. - fn logs(&self, filter: Filter) -> Vec; -} - -#[derive(Default, Clone, Debug, Eq, PartialEq)] /// Report on the status of a client. +#[derive(Default, Clone, Debug, Eq, PartialEq)] pub struct ClientReport { /// How many blocks have been imported so far. pub blocks_imported: usize, @@ -679,6 +566,8 @@ impl BlockChainClient for Client where V: Verifier { } fn logs(&self, filter: Filter) -> Vec { + // TODO: lock blockchain only once + let mut blocks = filter.bloom_possibilities().iter() .filter_map(|bloom| self.blocks_with_bloom(bloom, filter.from_block.clone(), filter.to_block.clone())) .flat_map(|m| m) diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs new file mode 100644 index 000000000..484c8d0c6 --- /dev/null +++ b/ethcore/src/client/config.rs @@ -0,0 +1,31 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +pub use block_queue::BlockQueueConfig; +pub use blockchain::BlockChainConfig; + +/// Client configuration. Includes configs for all sub-systems. +#[derive(Debug, Default)] +pub struct ClientConfig { + /// Block queue configuration. + pub queue: BlockQueueConfig, + /// Blockchain configuration. + pub blockchain: BlockChainConfig, + /// Prefer journal rather than archive. + pub prefer_journal: bool, + /// The name of the client instance. + pub name: String, +} diff --git a/ethcore/src/client/ids.rs b/ethcore/src/client/ids.rs new file mode 100644 index 000000000..303657a76 --- /dev/null +++ b/ethcore/src/client/ids.rs @@ -0,0 +1,44 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Unique identifiers. + +use util::hash::H256; +use header::BlockNumber; + +/// Uniquely identifies block. +#[derive(Debug, PartialEq, Clone)] +pub enum BlockId { + /// Block's sha3. + /// Querying by hash is always faster. + Hash(H256), + /// Block number within canon blockchain. + Number(BlockNumber), + /// Earliest block (genesis). + Earliest, + /// Latest mined block. + Latest +} + +/// Uniquely identifies transaction. +#[derive(Debug, PartialEq, Clone)] +pub enum TransactionId { + /// Transaction's sha3. + Hash(H256), + /// Block id and transaction index within this block. + /// Querying by block position is always faster. + Location(BlockId, usize) +} diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs new file mode 100644 index 000000000..0daa17553 --- /dev/null +++ b/ethcore/src/client/mod.rs @@ -0,0 +1,102 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Blockchain database client. + +mod client; +mod config; +mod ids; + +pub use self::client::*; +pub use self::config::{ClientConfig, BlockQueueConfig, BlockChainConfig}; +pub use self::ids::{BlockId, TransactionId}; + +use util::bytes::Bytes; +use util::hash::{Address, H256, H2048}; +use util::numbers::U256; +use blockchain::TreeRoute; +use block_queue::BlockQueueInfo; +use header::BlockNumber; +use transaction::LocalizedTransaction; +use log_entry::LocalizedLogEntry; +use filter::Filter; +use error::ImportResult; + +/// Blockchain database client. Owns and manages a blockchain and a block queue. +pub trait BlockChainClient : Sync + Send { + /// Get raw block header data by block id. + fn block_header(&self, id: BlockId) -> Option; + + /// Get raw block body data by block id. + /// Block body is an RLP list of two items: uncles and transactions. + fn block_body(&self, id: BlockId) -> Option; + + /// Get raw block data by block header hash. + fn block(&self, id: BlockId) -> Option; + + /// Get block status by block header hash. + fn block_status(&self, id: BlockId) -> BlockStatus; + + /// Get block total difficulty. + fn block_total_difficulty(&self, id: BlockId) -> Option; + + /// Get address nonce. + fn nonce(&self, address: &Address) -> U256; + + /// Get block hash. + fn block_hash(&self, id: BlockId) -> Option; + + /// Get address code. + fn code(&self, address: &Address) -> Option; + + /// Get transaction with given hash. + fn transaction(&self, id: TransactionId) -> Option; + + /// Get a tree route between `from` and `to`. + /// See `BlockChain::tree_route`. + fn tree_route(&self, from: &H256, to: &H256) -> Option; + + /// Get latest state node + fn state_data(&self, hash: &H256) -> Option; + + /// Get raw block receipts data by block header hash. + fn block_receipts(&self, hash: &H256) -> Option; + + /// Import a block into the blockchain. + fn import_block(&self, bytes: Bytes) -> ImportResult; + + /// Get block queue information. + fn queue_info(&self) -> BlockQueueInfo; + + /// Clear block queue and abort all import activity. + fn clear_queue(&self); + + /// Get blockchain information. + fn chain_info(&self) -> BlockChainInfo; + + /// Get the best block header. + fn best_block_header(&self) -> Bytes { + // TODO: lock blockchain only once + self.block_header(BlockId::Hash(self.chain_info().best_block_hash)).unwrap() + } + + /// Returns numbers of blocks containing given bloom. + fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockId, to_block: BlockId) -> Option>; + + /// Returns logs matching given filter. + fn logs(&self, filter: Filter) -> Vec; +} + From 027f122aea154b14c7563386251ff0407b4d2969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 10 Mar 2016 14:24:33 +0100 Subject: [PATCH 25/33] Removing get prefix from poll_info --- rpc/src/v1/helpers/poll_manager.rs | 18 +++++++++--------- rpc/src/v1/impls/eth.rs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rpc/src/v1/helpers/poll_manager.rs b/rpc/src/v1/helpers/poll_manager.rs index 36a6352c2..0297384d1 100644 --- a/rpc/src/v1/helpers/poll_manager.rs +++ b/rpc/src/v1/helpers/poll_manager.rs @@ -84,7 +84,7 @@ impl PollManager where T: Timer { } /// Returns number of block when last poll happend. - pub fn get_poll_info(&mut self, id: &PollId) -> Option<&PollInfo> { + pub fn poll_info(&mut self, id: &PollId) -> Option<&PollInfo> { self.polls.prune(); self.polls.get(id) } @@ -124,21 +124,21 @@ mod tests { *time.borrow_mut() = 10; indexer.update_poll(&0, 21); - assert_eq!(indexer.get_poll_info(&0).unwrap().filter, false); - assert_eq!(indexer.get_poll_info(&0).unwrap().block_number, 21); + assert_eq!(indexer.poll_info(&0).unwrap().filter, false); + assert_eq!(indexer.poll_info(&0).unwrap().block_number, 21); *time.borrow_mut() = 30; indexer.update_poll(&1, 23); - assert_eq!(indexer.get_poll_info(&1).unwrap().filter, true); - assert_eq!(indexer.get_poll_info(&1).unwrap().block_number, 23); + assert_eq!(indexer.poll_info(&1).unwrap().filter, true); + assert_eq!(indexer.poll_info(&1).unwrap().block_number, 23); *time.borrow_mut() = 75; indexer.update_poll(&0, 30); - assert!(indexer.get_poll_info(&0).is_none()); - assert_eq!(indexer.get_poll_info(&1).unwrap().filter, true); - assert_eq!(indexer.get_poll_info(&1).unwrap().block_number, 23); + assert!(indexer.poll_info(&0).is_none()); + assert_eq!(indexer.poll_info(&1).unwrap().filter, true); + assert_eq!(indexer.poll_info(&1).unwrap().block_number, 23); indexer.remove_poll(&1); - assert!(indexer.get_poll_info(&1).is_none()); + assert!(indexer.poll_info(&1).is_none()); } } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 7113c55b1..479bae95b 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -307,7 +307,7 @@ impl EthFilter for EthFilterClient { let client = take_weak!(self.client); from_params::<(Index,)>(params) .and_then(|(index,)| { - let info = self.polls.lock().unwrap().get_poll_info(&index.value()).cloned(); + let info = self.polls.lock().unwrap().poll_info(&index.value()).cloned(); match info { None => Ok(Value::Array(vec![] as Vec)), Some(info) => match info.filter { From 094ae4e9f9a77708c15751afa17c4835e1fb16a0 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 10 Mar 2016 19:15:10 +0400 Subject: [PATCH 26/33] personal is back to the master ver --- rpc/src/v1/impls/personal.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index a2788b9d9..48e1b1c6a 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -18,27 +18,28 @@ use std::sync::{Arc, Weak}; use jsonrpc_core::*; use v1::traits::Personal; +use util::keys::store::*; use util::Address; -use ethcore::client::Client; +use std::sync::RwLock; /// Account management (personal) rpc implementation. pub struct PersonalClient { - client: Weak, + secret_store: Weak>, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(client: &Arc) -> Self { + pub fn new(store: &Arc>) -> Self { PersonalClient { - client: Arc::downgrade(client), + secret_store: Arc::downgrade(store), } } } impl Personal for PersonalClient { fn accounts(&self, _: Params) -> Result { - let client = take_weak!(self.client); - let store = client.secret_store().read().unwrap(); + let store_wk = take_weak!(self.secret_store); + let store = store_wk.read().unwrap(); match store.accounts() { Ok(account_list) => { Ok(Value::Array(account_list.iter() @@ -53,8 +54,8 @@ impl Personal for PersonalClient { fn new_account(&self, params: Params) -> Result { from_params::<(String, )>(params).and_then( |(pass, )| { - let client = take_weak!(self.client); - let mut store = client.secret_store().write().unwrap(); + let store_wk = take_weak!(self.secret_store); + let mut store = store_wk.write().unwrap(); match store.new_account(&pass) { Ok(address) => Ok(Value::String(format!("{:?}", address))), Err(_) => Err(Error::internal_error()) @@ -66,8 +67,8 @@ impl Personal for PersonalClient { fn unlock_account(&self, params: Params) -> Result { from_params::<(Address, String, u64)>(params).and_then( |(account, account_pass, _)|{ - let client = take_weak!(self.client); - let store = client.secret_store().read().unwrap(); + let store_wk = take_weak!(self.secret_store); + let store = store_wk.read().unwrap(); match store.unlock_account(&account, &account_pass) { Ok(_) => Ok(Value::Bool(true)), Err(_) => Ok(Value::Bool(false)), From c951dee7668e4d6ef5164230bdc1d41b77e4254e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 10 Mar 2016 17:09:34 +0100 Subject: [PATCH 27/33] --archive is default. --pruning is option. --- parity/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 68d45bc04..69650270a 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -79,7 +79,7 @@ Protocol Options: or olympic, frontier, homestead, mainnet, morden, or testnet [default: homestead]. --testnet Equivalent to --chain testnet (geth-compatible). --networkid INDEX Override the network identifier from the chain we are on. - --archive Client should not prune the state/storage trie. + --pruning Client should prune the state/storage trie. -d --datadir PATH Specify the database & configuration directory path [default: $HOME/.parity] --keys-path PATH Specify the path for JSON key files to be found [default: $HOME/.web3/keys] --identity NAME Specify your node's name. @@ -140,7 +140,7 @@ struct Args { flag_identity: String, flag_cache: Option, flag_keys_path: String, - flag_archive: bool, + flag_pruning: bool, flag_no_bootstrap: bool, flag_listen_address: String, flag_public_address: Option, @@ -402,7 +402,7 @@ impl Configuration { client_config.blockchain.max_cache_size = self.args.flag_cache_max_size; } } - client_config.prefer_journal = !self.args.flag_archive; + client_config.prefer_journal = self.args.flag_pruning; client_config.name = self.args.flag_identity.clone(); client_config.queue.max_mem_use = self.args.flag_queue_max_size; let mut service = ClientService::start(client_config, spec, net_settings, &Path::new(&self.path())).unwrap(); From 5571503c224b5d1b185243d7f5e6f1f1bc3a6856 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 10 Mar 2016 20:18:01 +0400 Subject: [PATCH 28/33] traitified secret store --- rpc/src/v1/impls/eth.rs | 16 +++++++++------- util/src/keys/store.rs | 22 ++++++++++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 4a8461c45..97d248ef6 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -30,20 +30,23 @@ use ethcore::ethereum::denominations::shannon; use v1::traits::{Eth, EthFilter}; use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncInfo, Transaction, TransactionRequest, OptionalValue, Index, Filter, Log}; use v1::helpers::{PollFilter, PollManager}; +use util::keys::store::AccountProvider; /// Eth rpc implementation. -pub struct EthClient where C: BlockChainClient, S: SyncStatusProvider { +pub struct EthClient where C: BlockChainClient, S: SyncStatusProvider, A: AccountProvider { client: Weak, sync: Weak, + accounts: Weak, hashrates: RwLock>, } -impl EthClient where C: BlockChainClient, S: SyncStatusProvider { +impl EthClient where C: BlockChainClient, S: SyncStatusProvider, A: AccountProvider { /// Creates new EthClient. - pub fn new(client: &Arc, sync: &Arc) -> Self { + pub fn new(client: &Arc, sync: &Arc, accounts: &Arc) -> Self { EthClient { client: Arc::downgrade(client), sync: Arc::downgrade(sync), + accounts: Arc::downgrade(accounts), hashrates: RwLock::new(HashMap::new()), } } @@ -94,7 +97,7 @@ impl EthClient where C: BlockChainClient, S: SyncStatusProvider { } } -impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncStatusProvider + 'static { +impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncStatusProvider + 'static, A: AccountProvider + 'static { fn protocol_version(&self, params: Params) -> Result { match params { Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), @@ -256,9 +259,8 @@ impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncS fn send_transaction(&self, params: Params) -> Result { from_params::<(TransactionRequest, )>(params) .and_then(|(transaction_request, )| { - let client = take_weak!(self.client); - let store = client.secret_store().read().unwrap(); - match store.account_secret(&transaction_request.from) { + let accounts = take_weak!(self.accounts); + match accounts.account_secret(&transaction_request.from) { Ok(secret) => { let sync = take_weak!(self.sync); let (transaction, _) = transaction_request.to_eth(); diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index dcc165259..9ea00cbba 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -78,6 +78,18 @@ struct AccountUnlock { expires: DateTime, } +/// Basic account management trait +pub trait AccountProvider : Send + Sync { + /// Unlocks account with the password provided + fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError>; + /// Creates account + fn new_account(&mut self, pass: &str) -> Result; + /// Returns secret for unlocked account + fn account_secret(&self, account: &Address) -> Result; + /// Returns secret for unlocked account + fn sign(&self, account: &Address, message: &H256) -> Result; +} + impl SecretStore { /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { @@ -144,9 +156,11 @@ impl SecretStore { unlocks: RwLock::new(HashMap::new()), } } +} +impl AccountProvider for SecretStore { /// Unlocks account for use - pub fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { + fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { let secret_id = try!(self.account(&account).ok_or(EncryptedHashMapError::UnknownIdentifier)); let secret = try!(self.get(&secret_id, pass)); { @@ -160,7 +174,7 @@ impl SecretStore { } /// Creates new account - pub fn new_account(&mut self, pass: &str) -> Result { + fn new_account(&mut self, pass: &str) -> Result { let secret = H256::random(); let key_id = H128::random(); self.insert(key_id.clone(), secret, pass); @@ -173,7 +187,7 @@ impl SecretStore { } /// Signs message with unlocked account - pub fn sign(&self, account: &Address, message: &H256) -> Result { + fn sign(&self, account: &Address, message: &H256) -> Result { let read_lock = self.unlocks.read().unwrap(); let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); match crypto::KeyPair::from_secret(unlock.secret) { @@ -186,7 +200,7 @@ impl SecretStore { } /// Returns secret for unlocked account - pub fn account_secret(&self, account: &Address) -> Result { + fn account_secret(&self, account: &Address) -> Result { let read_lock = self.unlocks.read().unwrap(); let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); Ok(unlock.secret as crypto::Secret) From 25a63611f856ab0896b114b6388461c174bf9905 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 10 Mar 2016 20:32:17 +0400 Subject: [PATCH 29/33] extend sync status interface to sync provider --- parity/main.rs | 2 +- rpc/src/v1/impls/eth.rs | 8 ++++---- rpc/src/v1/impls/net.rs | 8 ++++---- sync/src/lib.rs | 20 +++++++++++--------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 8814da44b..d2b9b3567 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -49,7 +49,7 @@ use ethcore::spec::*; use ethcore::client::*; use ethcore::service::{ClientService, NetSyncMessage}; use ethcore::ethereum; -use ethsync::{EthSync, SyncConfig, SyncStatusProvider}; +use ethsync::{EthSync, SyncConfig, SyncProvider}; use docopt::Docopt; use daemonize::Daemonize; use number_prefix::{binary_prefix, Standalone, Prefixed}; diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 91bd0cce3..a067b48fb 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -17,7 +17,7 @@ //! Eth rpc implementation. use std::collections::HashMap; use std::sync::{Arc, Weak, Mutex, RwLock}; -use ethsync::{SyncStatusProvider, SyncState}; +use ethsync::{SyncProvider, SyncState}; use jsonrpc_core::*; use util::numbers::*; use util::sha3::*; @@ -32,13 +32,13 @@ use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncIn use v1::helpers::{PollFilter, PollManager}; /// Eth rpc implementation. -pub struct EthClient where C: BlockChainClient, S: SyncStatusProvider { +pub struct EthClient where C: BlockChainClient, S: SyncProvider { client: Weak, sync: Weak, hashrates: RwLock>, } -impl EthClient where C: BlockChainClient, S: SyncStatusProvider { +impl EthClient where C: BlockChainClient, S: SyncProvider { /// Creates new EthClient. pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { @@ -94,7 +94,7 @@ impl EthClient where C: BlockChainClient, S: SyncStatusProvider { } } -impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncStatusProvider + 'static { +impl Eth for EthClient where C: BlockChainClient + 'static, S: SyncProvider + 'static { fn protocol_version(&self, params: Params) -> Result { match params { Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), diff --git a/rpc/src/v1/impls/net.rs b/rpc/src/v1/impls/net.rs index a686ed66f..5e67bf252 100644 --- a/rpc/src/v1/impls/net.rs +++ b/rpc/src/v1/impls/net.rs @@ -17,15 +17,15 @@ //! Net rpc implementation. use std::sync::{Arc, Weak}; use jsonrpc_core::*; -use ethsync::SyncStatusProvider; +use ethsync::SyncProvider; use v1::traits::Net; /// Net rpc implementation. -pub struct NetClient where S: SyncStatusProvider { +pub struct NetClient where S: SyncProvider { sync: Weak } -impl NetClient where S: SyncStatusProvider { +impl NetClient where S: SyncProvider { /// Creates new NetClient. pub fn new(sync: &Arc) -> Self { NetClient { @@ -34,7 +34,7 @@ impl NetClient where S: SyncStatusProvider { } } -impl Net for NetClient where S: SyncStatusProvider + 'static { +impl Net for NetClient where S: SyncProvider + 'static { fn version(&self, _: Params) -> Result { Ok(Value::U64(take_weak!(self.sync).status().protocol_version as u64)) } diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 5a0ba79c5..3b79e5614 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -95,9 +95,11 @@ impl Default for SyncConfig { } /// Current sync status -pub trait SyncStatusProvider: Send + Sync { +pub trait SyncProvider: Send + Sync { /// Get sync status fn status(&self) -> SyncStatus; + /// Insert transaction in the sync transaction queue + fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction); } /// Ethereum network protocol handler @@ -130,9 +132,16 @@ impl EthSync { pub fn restart(&mut self, io: &mut NetworkContext) { self.sync.write().unwrap().restart(&mut NetSyncIo::new(io, self.chain.deref())); } +} + +impl SyncProvider for EthSync { + /// Get sync status + fn status(&self) -> SyncStatus { + self.sync.read().unwrap().status() + } /// Insert transaction in transaction queue - pub fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction) { + fn insert_transaction(&self, transaction: ethcore::transaction::SignedTransaction) { use util::numbers::*; let nonce_fn = |a: &Address| self.chain.state().nonce(a) + U256::one(); @@ -141,13 +150,6 @@ impl EthSync { } } -impl SyncStatusProvider for EthSync { - /// Get sync status - fn status(&self) -> SyncStatus { - self.sync.read().unwrap().status() - } -} - impl NetworkProtocolHandler for EthSync { fn initialize(&self, io: &NetworkContext) { io.register_timer(0, 1000).expect("Error registering sync timer"); From a2dea3885b393da47dad0127f22773b7fcae00bc Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 10 Mar 2016 23:09:45 +0400 Subject: [PATCH 30/33] refactoring to AccountService --- parity/main.rs | 10 ++----- rpc/src/v1/impls/personal.rs | 18 +++++------- util/src/keys/store.rs | 53 +++++++++++++++++++++++++++++++----- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 6d1c08162..b6ed5cba3 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -196,7 +196,7 @@ fn setup_log(init: &Option) { } #[cfg(feature = "rpc")] -fn setup_rpc_server(client: Arc, sync: Arc, secret_store: Arc, url: &str, cors_domain: &str, apis: Vec<&str>) -> Option> { +fn setup_rpc_server(client: Arc, sync: Arc, secret_store: Arc, url: &str, cors_domain: &str, apis: Vec<&str>) -> Option> { use rpc::v1::*; let server = rpc::RpcServer::new(); @@ -416,11 +416,7 @@ impl Configuration { let sync = EthSync::register(service.network(), sync_config, client); // Secret Store - let secret_store = Arc::new(SecretStore::new()); - { - let import_ref = Arc::make_mut(&mut secret_store); - import_ref.try_import_existing(); - } + let account_service = Arc::new(AccountService::new()); // Setup rpc if self.args.flag_jsonrpc || self.args.flag_rpc { @@ -432,7 +428,7 @@ impl Configuration { let cors = self.args.flag_rpccorsdomain.as_ref().unwrap_or(&self.args.flag_jsonrpc_cors); // TODO: use this as the API list. let apis = self.args.flag_rpcapi.as_ref().unwrap_or(&self.args.flag_jsonrpc_apis); - let server_handler = setup_rpc_server(service.client(), sync.clone(), secret_store.clone(), &url, cors, apis.split(",").collect()); + let server_handler = setup_rpc_server(service.client(), sync.clone(), account_service.clone(), &url, cors, apis.split(",").collect()); if let Some(handler) = server_handler { panic_handler.forward_from(handler.deref()); } diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index 48e1b1c6a..7b79ceae7 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -20,30 +20,28 @@ use jsonrpc_core::*; use v1::traits::Personal; use util::keys::store::*; use util::Address; -use std::sync::RwLock; /// Account management (personal) rpc implementation. pub struct PersonalClient { - secret_store: Weak>, + accounts: Weak, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(store: &Arc>) -> Self { + pub fn new(store: &Arc) -> Self { PersonalClient { - secret_store: Arc::downgrade(store), + accounts: Arc::downgrade(store), } } } impl Personal for PersonalClient { fn accounts(&self, _: Params) -> Result { - let store_wk = take_weak!(self.secret_store); - let store = store_wk.read().unwrap(); + let store = take_weak!(self.accounts); match store.accounts() { Ok(account_list) => { Ok(Value::Array(account_list.iter() - .map(|&(account, _)| Value::String(format!("{:?}", account))) + .map(|&account| Value::String(format!("{:?}", account))) .collect::>()) ) } @@ -54,8 +52,7 @@ impl Personal for PersonalClient { fn new_account(&self, params: Params) -> Result { from_params::<(String, )>(params).and_then( |(pass, )| { - let store_wk = take_weak!(self.secret_store); - let mut store = store_wk.write().unwrap(); + let store = take_weak!(self.accounts); match store.new_account(&pass) { Ok(address) => Ok(Value::String(format!("{:?}", address))), Err(_) => Err(Error::internal_error()) @@ -67,8 +64,7 @@ impl Personal for PersonalClient { fn unlock_account(&self, params: Params) -> Result { from_params::<(Address, String, u64)>(params).and_then( |(account, account_pass, _)|{ - let store_wk = take_weak!(self.secret_store); - let store = store_wk.read().unwrap(); + let store = take_weak!(self.accounts); match store.unlock_account(&account, &account_pass) { Ok(_) => Ok(Value::Bool(true)), Err(_) => Ok(Value::Bool(false)), diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 9ea00cbba..ea97cc80e 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -80,16 +80,57 @@ struct AccountUnlock { /// Basic account management trait pub trait AccountProvider : Send + Sync { + /// Lists all accounts + fn accounts(&self) -> Result, ::std::io::Error>; /// Unlocks account with the password provided fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError>; /// Creates account - fn new_account(&mut self, pass: &str) -> Result; + fn new_account(&self, pass: &str) -> Result; /// Returns secret for unlocked account fn account_secret(&self, account: &Address) -> Result; /// Returns secret for unlocked account fn sign(&self, account: &Address, message: &H256) -> Result; } +/// Thread-safe accounts management +pub struct AccountService { + secret_store: RwLock, +} + +impl AccountProvider for AccountService { + /// Lists all accounts + fn accounts(&self) -> Result, ::std::io::Error> { + Ok(try!(self.secret_store.read().unwrap().accounts()).iter().map(|&(addr, _)| addr).collect::>()) + } + /// Unlocks account with the password provided + fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { + self.secret_store.read().unwrap().unlock_account(account, pass) + } + /// Creates account + fn new_account(&self, pass: &str) -> Result { + self.secret_store.write().unwrap().new_account(pass) + } + /// Returns secret for unlocked account + fn account_secret(&self, account: &Address) -> Result { + self.secret_store.read().unwrap().account_secret(account) + } + /// Returns secret for unlocked account + fn sign(&self, account: &Address, message: &H256) -> Result { + self.secret_store.read().unwrap().sign(account, message) + } +} + +impl AccountService { + /// New account service with the default location + pub fn new() -> AccountService { + let secret_store = RwLock::new(SecretStore::new()); + secret_store.write().unwrap().try_import_existing(); + AccountService { + secret_store: secret_store + } + } +} + impl SecretStore { /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { @@ -156,11 +197,9 @@ impl SecretStore { unlocks: RwLock::new(HashMap::new()), } } -} -impl AccountProvider for SecretStore { /// Unlocks account for use - fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { + pub fn unlock_account(&self, account: &Address, pass: &str) -> Result<(), EncryptedHashMapError> { let secret_id = try!(self.account(&account).ok_or(EncryptedHashMapError::UnknownIdentifier)); let secret = try!(self.get(&secret_id, pass)); { @@ -174,7 +213,7 @@ impl AccountProvider for SecretStore { } /// Creates new account - fn new_account(&mut self, pass: &str) -> Result { + pub fn new_account(&mut self, pass: &str) -> Result { let secret = H256::random(); let key_id = H128::random(); self.insert(key_id.clone(), secret, pass); @@ -187,7 +226,7 @@ impl AccountProvider for SecretStore { } /// Signs message with unlocked account - fn sign(&self, account: &Address, message: &H256) -> Result { + pub fn sign(&self, account: &Address, message: &H256) -> Result { let read_lock = self.unlocks.read().unwrap(); let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); match crypto::KeyPair::from_secret(unlock.secret) { @@ -200,7 +239,7 @@ impl AccountProvider for SecretStore { } /// Returns secret for unlocked account - fn account_secret(&self, account: &Address) -> Result { + pub fn account_secret(&self, account: &Address) -> Result { let read_lock = self.unlocks.read().unwrap(); let unlock = try!(read_lock.get(account).ok_or(SigningError::AccountNotUnlocked)); Ok(unlock.secret as crypto::Secret) From c5c8851b5097e27ccadc43b579dd8f605b9c9733 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Mar 2016 20:27:50 +0100 Subject: [PATCH 31/33] moved TestBlockChainClient to ethcore --- ethcore/src/client/mod.rs | 2 + ethcore/src/client/test_client.rs | 322 ++++++++++++++++++++++++++++++ ethcore/src/lib.rs | 2 +- sync/src/tests/chain.rs | 2 +- sync/src/tests/helpers.rs | 292 +-------------------------- 5 files changed, 327 insertions(+), 293 deletions(-) create mode 100644 ethcore/src/client/test_client.rs diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 0daa17553..58a21f151 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -19,10 +19,12 @@ mod client; mod config; mod ids; +mod test_client; pub use self::client::*; pub use self::config::{ClientConfig, BlockQueueConfig, BlockChainConfig}; pub use self::ids::{BlockId, TransactionId}; +pub use self::test_client::{TestBlockChainClient, EachBlockWith}; use util::bytes::Bytes; use util::hash::{Address, H256, H2048}; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs new file mode 100644 index 000000000..4ca30dcd5 --- /dev/null +++ b/ethcore/src/client/test_client.rs @@ -0,0 +1,322 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//use std::mem; +//use std::ops::{Deref, DerefMut}; +//use std::collections::HashMap; +//use rustc_serialize::hex::FromHex; +//use util::rlp; +//use util::rlp::*; +//use util::bytes::Bytes; +//use util::hash::{FixedHash, Address, H256, H2048}; +//use util::numbers::{Uint, U256}; +//use util::crypto::KeyPair; +//use util::sha3::Hashable; +use util::*; +//use std::sync::RwLock; +use transaction::{Transaction, LocalizedTransaction, Action}; +use blockchain::TreeRoute; +use client::{BlockChainClient, BlockChainInfo, BlockStatus, BlockId, TransactionId}; +use header::{Header as BlockHeader, BlockNumber}; +use filter::Filter; +use log_entry::LocalizedLogEntry; +use receipt::Receipt; +use error::ImportResult; +use block_queue::BlockQueueInfo; + +pub struct TestBlockChainClient { + pub blocks: RwLock>, + pub numbers: RwLock>, + pub genesis_hash: H256, + pub last_hash: RwLock, + pub difficulty: RwLock, +} + +#[derive(Clone)] +pub enum EachBlockWith { + Nothing, + Uncle, + Transaction, + UncleAndTransaction +} + +impl TestBlockChainClient { + pub fn new() -> TestBlockChainClient { + + let mut client = TestBlockChainClient { + blocks: RwLock::new(HashMap::new()), + numbers: RwLock::new(HashMap::new()), + genesis_hash: H256::new(), + last_hash: RwLock::new(H256::new()), + difficulty: RwLock::new(From::from(0)), + }; + client.add_blocks(1, EachBlockWith::Nothing); // add genesis block + client.genesis_hash = client.last_hash.read().unwrap().clone(); + client + } + + pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { + let len = self.numbers.read().unwrap().len(); + for n in len..(len + count) { + let mut header = BlockHeader::new(); + header.difficulty = From::from(n); + header.parent_hash = self.last_hash.read().unwrap().clone(); + header.number = n as BlockNumber; + let uncles = match with { + EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => { + let mut uncles = RlpStream::new_list(1); + let mut uncle_header = BlockHeader::new(); + uncle_header.difficulty = From::from(n); + uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); + uncle_header.number = n as BlockNumber; + uncles.append(&uncle_header); + header.uncles_hash = uncles.as_raw().sha3(); + uncles + }, + _ => RlpStream::new_list(0) + }; + let txs = match with { + EachBlockWith::Transaction | EachBlockWith::UncleAndTransaction => { + let mut txs = RlpStream::new_list(1); + let keypair = KeyPair::create().unwrap(); + let tx = Transaction { + action: Action::Create, + value: U256::from(100), + data: "3331600055".from_hex().unwrap(), + gas: U256::from(100_000), + gas_price: U256::one(), + nonce: U256::zero() + }; + let signed_tx = tx.sign(&keypair.secret()); + txs.append(&signed_tx); + txs.out() + }, + _ => rlp::NULL_RLP.to_vec() + }; + + let mut rlp = RlpStream::new_list(3); + rlp.append(&header); + rlp.append_raw(&txs, 1); + rlp.append_raw(uncles.as_raw(), 1); + self.import_block(rlp.as_raw().to_vec()).unwrap(); + } + } + + pub fn corrupt_block(&mut self, n: BlockNumber) { + let hash = self.block_hash(BlockId::Number(n)).unwrap(); + let mut header: BlockHeader = decode(&self.block_header(BlockId::Number(n)).unwrap()); + header.parent_hash = H256::new(); + let mut rlp = RlpStream::new_list(3); + rlp.append(&header); + rlp.append_raw(&rlp::NULL_RLP, 1); + rlp.append_raw(&rlp::NULL_RLP, 1); + self.blocks.write().unwrap().insert(hash, rlp.out()); + } + + pub fn block_hash_delta_minus(&mut self, delta: usize) -> H256 { + let blocks_read = self.numbers.read().unwrap(); + let index = blocks_read.len() - delta; + blocks_read[&index].clone() + } + + fn block_hash(&self, id: BlockId) -> Option { + match id { + BlockId::Hash(hash) => Some(hash), + BlockId::Number(n) => self.numbers.read().unwrap().get(&(n as usize)).cloned(), + BlockId::Earliest => self.numbers.read().unwrap().get(&0).cloned(), + BlockId::Latest => self.numbers.read().unwrap().get(&(self.numbers.read().unwrap().len() - 1)).cloned() + } + } +} + +impl BlockChainClient for TestBlockChainClient { + fn block_total_difficulty(&self, _id: BlockId) -> Option { + Some(U256::zero()) + } + + fn block_hash(&self, _id: BlockId) -> Option { + unimplemented!(); + } + + fn nonce(&self, _address: &Address) -> U256 { + U256::zero() + } + + fn code(&self, _address: &Address) -> Option { + unimplemented!(); + } + + fn transaction(&self, _id: TransactionId) -> Option { + unimplemented!(); + } + + fn blocks_with_bloom(&self, _bloom: &H2048, _from_block: BlockId, _to_block: BlockId) -> Option> { + unimplemented!(); + } + + fn logs(&self, _filter: Filter) -> Vec { + unimplemented!(); + } + + fn block_header(&self, id: BlockId) -> Option { + self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).map(|r| Rlp::new(r).at(0).as_raw().to_vec())) + } + + fn block_body(&self, id: BlockId) -> Option { + self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).map(|r| { + let mut stream = RlpStream::new_list(2); + stream.append_raw(Rlp::new(&r).at(1).as_raw(), 1); + stream.append_raw(Rlp::new(&r).at(2).as_raw(), 1); + stream.out() + })) + } + + fn block(&self, id: BlockId) -> Option { + self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).cloned()) + } + + fn block_status(&self, id: BlockId) -> BlockStatus { + match id { + BlockId::Number(number) if (number as usize) < self.blocks.read().unwrap().len() => BlockStatus::InChain, + BlockId::Hash(ref hash) if self.blocks.read().unwrap().get(hash).is_some() => BlockStatus::InChain, + _ => BlockStatus::Unknown + } + } + + // works only if blocks are one after another 1 -> 2 -> 3 + fn tree_route(&self, from: &H256, to: &H256) -> Option { + Some(TreeRoute { + ancestor: H256::new(), + index: 0, + blocks: { + let numbers_read = self.numbers.read().unwrap(); + let mut adding = false; + + let mut blocks = Vec::new(); + for (_, hash) in numbers_read.iter().sort_by(|tuple1, tuple2| tuple1.0.cmp(tuple2.0)) { + if hash == to { + if adding { + blocks.push(hash.clone()); + } + adding = false; + break; + } + if hash == from { + adding = true; + } + if adding { + blocks.push(hash.clone()); + } + } + if adding { Vec::new() } else { blocks } + } + }) + } + + // TODO: returns just hashes instead of node state rlp(?) + fn state_data(&self, hash: &H256) -> Option { + // starts with 'f' ? + if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { + let mut rlp = RlpStream::new(); + rlp.append(&hash.clone()); + return Some(rlp.out()); + } + None + } + + fn block_receipts(&self, hash: &H256) -> Option { + // starts with 'f' ? + if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { + let receipt = Receipt::new( + H256::zero(), + U256::zero(), + vec![]); + let mut rlp = RlpStream::new(); + rlp.append(&receipt); + return Some(rlp.out()); + } + None + } + + fn import_block(&self, b: Bytes) -> ImportResult { + let header = Rlp::new(&b).val_at::(0); + let h = header.hash(); + let number: usize = header.number as usize; + if number > self.blocks.read().unwrap().len() { + panic!("Unexpected block number. Expected {}, got {}", self.blocks.read().unwrap().len(), number); + } + if number > 0 { + match self.blocks.read().unwrap().get(&header.parent_hash) { + Some(parent) => { + let parent = Rlp::new(parent).val_at::(0); + if parent.number != (header.number - 1) { + panic!("Unexpected block parent"); + } + }, + None => { + panic!("Unknown block parent {:?} for block {}", header.parent_hash, number); + } + } + } + let len = self.numbers.read().unwrap().len(); + if number == len { + { + let mut difficulty = self.difficulty.write().unwrap(); + *difficulty.deref_mut() = *difficulty.deref() + header.difficulty; + } + mem::replace(self.last_hash.write().unwrap().deref_mut(), h.clone()); + self.blocks.write().unwrap().insert(h.clone(), b); + self.numbers.write().unwrap().insert(number, h.clone()); + let mut parent_hash = header.parent_hash; + if number > 0 { + let mut n = number - 1; + while n > 0 && self.numbers.read().unwrap()[&n] != parent_hash { + *self.numbers.write().unwrap().get_mut(&n).unwrap() = parent_hash.clone(); + n -= 1; + parent_hash = Rlp::new(&self.blocks.read().unwrap()[&parent_hash]).val_at::(0).parent_hash; + } + } + } + else { + self.blocks.write().unwrap().insert(h.clone(), b.to_vec()); + } + Ok(h) + } + + fn queue_info(&self) -> BlockQueueInfo { + BlockQueueInfo { + verified_queue_size: 0, + unverified_queue_size: 0, + verifying_queue_size: 0, + max_queue_size: 0, + max_mem_use: 0, + mem_used: 0, + } + } + + fn clear_queue(&self) { + } + + fn chain_info(&self) -> BlockChainInfo { + BlockChainInfo { + total_difficulty: *self.difficulty.read().unwrap(), + pending_total_difficulty: *self.difficulty.read().unwrap(), + genesis_hash: self.genesis_hash.clone(), + best_block_hash: self.last_hash.read().unwrap().clone(), + best_block_number: self.blocks.read().unwrap().len() as BlockNumber - 1, + } + } +} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 469364eb3..0ff5c1903 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -86,6 +86,7 @@ extern crate crossbeam; #[cfg(feature = "jit" )] extern crate evmjit; pub mod block; +pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -119,7 +120,6 @@ mod substate; mod executive; mod externalities; mod verification; -mod block_queue; mod blockchain; #[cfg(test)] diff --git a/sync/src/tests/chain.rs b/sync/src/tests/chain.rs index 855aa79a6..eebbdb164 100644 --- a/sync/src/tests/chain.rs +++ b/sync/src/tests/chain.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use util::*; -use ethcore::client::{BlockChainClient, BlockId}; +use ethcore::client::{BlockChainClient, BlockId, EachBlockWith}; use io::SyncIo; use chain::{SyncState}; use super::helpers::*; diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index d01dba0b2..ca4ae5158 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -15,300 +15,10 @@ // along with Parity. If not, see . use util::*; -use ethcore::client::{BlockChainClient, BlockStatus, TreeRoute, BlockChainInfo, TransactionId, BlockId, BlockQueueInfo}; -use ethcore::header::{Header as BlockHeader, BlockNumber}; -use ethcore::error::*; +use ethcore::client::{TestBlockChainClient, BlockChainClient}; use io::SyncIo; use chain::ChainSync; use ::SyncConfig; -use ethcore::receipt::Receipt; -use ethcore::transaction::{LocalizedTransaction, Transaction, Action}; -use ethcore::filter::Filter; -use ethcore::log_entry::LocalizedLogEntry; - -pub struct TestBlockChainClient { - pub blocks: RwLock>, - pub numbers: RwLock>, - pub genesis_hash: H256, - pub last_hash: RwLock, - pub difficulty: RwLock, -} - -#[derive(Clone)] -pub enum EachBlockWith { - Nothing, - Uncle, - Transaction, - UncleAndTransaction -} - -impl TestBlockChainClient { - pub fn new() -> TestBlockChainClient { - - let mut client = TestBlockChainClient { - blocks: RwLock::new(HashMap::new()), - numbers: RwLock::new(HashMap::new()), - genesis_hash: H256::new(), - last_hash: RwLock::new(H256::new()), - difficulty: RwLock::new(From::from(0)), - }; - client.add_blocks(1, EachBlockWith::Nothing); // add genesis block - client.genesis_hash = client.last_hash.read().unwrap().clone(); - client - } - - pub fn add_blocks(&mut self, count: usize, with: EachBlockWith) { - let len = self.numbers.read().unwrap().len(); - for n in len..(len + count) { - let mut header = BlockHeader::new(); - header.difficulty = From::from(n); - header.parent_hash = self.last_hash.read().unwrap().clone(); - header.number = n as BlockNumber; - let uncles = match with { - EachBlockWith::Uncle | EachBlockWith::UncleAndTransaction => { - let mut uncles = RlpStream::new_list(1); - let mut uncle_header = BlockHeader::new(); - uncle_header.difficulty = From::from(n); - uncle_header.parent_hash = self.last_hash.read().unwrap().clone(); - uncle_header.number = n as BlockNumber; - uncles.append(&uncle_header); - header.uncles_hash = uncles.as_raw().sha3(); - uncles - }, - _ => RlpStream::new_list(0) - }; - let txs = match with { - EachBlockWith::Transaction | EachBlockWith::UncleAndTransaction => { - let mut txs = RlpStream::new_list(1); - let keypair = KeyPair::create().unwrap(); - let tx = Transaction { - action: Action::Create, - value: U256::from(100), - data: "3331600055".from_hex().unwrap(), - gas: U256::from(100_000), - gas_price: U256::one(), - nonce: U256::zero() - }; - let signed_tx = tx.sign(&keypair.secret()); - txs.append(&signed_tx); - txs.out() - }, - _ => rlp::NULL_RLP.to_vec() - }; - - let mut rlp = RlpStream::new_list(3); - rlp.append(&header); - rlp.append_raw(&txs, 1); - rlp.append_raw(uncles.as_raw(), 1); - self.import_block(rlp.as_raw().to_vec()).unwrap(); - } - } - - pub fn corrupt_block(&mut self, n: BlockNumber) { - let hash = self.block_hash(BlockId::Number(n)).unwrap(); - let mut header: BlockHeader = decode(&self.block_header(BlockId::Number(n)).unwrap()); - header.parent_hash = H256::new(); - let mut rlp = RlpStream::new_list(3); - rlp.append(&header); - rlp.append_raw(&rlp::NULL_RLP, 1); - rlp.append_raw(&rlp::NULL_RLP, 1); - self.blocks.write().unwrap().insert(hash, rlp.out()); - } - - pub fn block_hash_delta_minus(&mut self, delta: usize) -> H256 { - let blocks_read = self.numbers.read().unwrap(); - let index = blocks_read.len() - delta; - blocks_read[&index].clone() - } - - fn block_hash(&self, id: BlockId) -> Option { - match id { - BlockId::Hash(hash) => Some(hash), - BlockId::Number(n) => self.numbers.read().unwrap().get(&(n as usize)).cloned(), - BlockId::Earliest => self.numbers.read().unwrap().get(&0).cloned(), - BlockId::Latest => self.numbers.read().unwrap().get(&(self.numbers.read().unwrap().len() - 1)).cloned() - } - } -} - -impl BlockChainClient for TestBlockChainClient { - fn block_total_difficulty(&self, _id: BlockId) -> Option { - Some(U256::zero()) - } - - fn block_hash(&self, _id: BlockId) -> Option { - unimplemented!(); - } - - fn nonce(&self, _address: &Address) -> U256 { - U256::zero() - } - - fn code(&self, _address: &Address) -> Option { - unimplemented!(); - } - - fn transaction(&self, _id: TransactionId) -> Option { - unimplemented!(); - } - - fn blocks_with_bloom(&self, _bloom: &H2048, _from_block: BlockId, _to_block: BlockId) -> Option> { - unimplemented!(); - } - - fn logs(&self, _filter: Filter) -> Vec { - unimplemented!(); - } - - fn block_header(&self, id: BlockId) -> Option { - self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).map(|r| Rlp::new(r).at(0).as_raw().to_vec())) - } - - fn block_body(&self, id: BlockId) -> Option { - self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).map(|r| { - let mut stream = RlpStream::new_list(2); - stream.append_raw(Rlp::new(&r).at(1).as_raw(), 1); - stream.append_raw(Rlp::new(&r).at(2).as_raw(), 1); - stream.out() - })) - } - - fn block(&self, id: BlockId) -> Option { - self.block_hash(id).and_then(|hash| self.blocks.read().unwrap().get(&hash).cloned()) - } - - fn block_status(&self, id: BlockId) -> BlockStatus { - match id { - BlockId::Number(number) if (number as usize) < self.blocks.read().unwrap().len() => BlockStatus::InChain, - BlockId::Hash(ref hash) if self.blocks.read().unwrap().get(hash).is_some() => BlockStatus::InChain, - _ => BlockStatus::Unknown - } - } - - // works only if blocks are one after another 1 -> 2 -> 3 - fn tree_route(&self, from: &H256, to: &H256) -> Option { - Some(TreeRoute { - ancestor: H256::new(), - index: 0, - blocks: { - let numbers_read = self.numbers.read().unwrap(); - let mut adding = false; - - let mut blocks = Vec::new(); - for (_, hash) in numbers_read.iter().sort_by(|tuple1, tuple2| tuple1.0.cmp(tuple2.0)) { - if hash == to { - if adding { - blocks.push(hash.clone()); - } - adding = false; - break; - } - if hash == from { - adding = true; - } - if adding { - blocks.push(hash.clone()); - } - } - if adding { Vec::new() } else { blocks } - } - }) - } - - // TODO: returns just hashes instead of node state rlp(?) - fn state_data(&self, hash: &H256) -> Option { - // starts with 'f' ? - if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { - let mut rlp = RlpStream::new(); - rlp.append(&hash.clone()); - return Some(rlp.out()); - } - None - } - - fn block_receipts(&self, hash: &H256) -> Option { - // starts with 'f' ? - if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { - let receipt = Receipt::new( - H256::zero(), - U256::zero(), - vec![]); - let mut rlp = RlpStream::new(); - rlp.append(&receipt); - return Some(rlp.out()); - } - None - } - - fn import_block(&self, b: Bytes) -> ImportResult { - let header = Rlp::new(&b).val_at::(0); - let h = header.hash(); - let number: usize = header.number as usize; - if number > self.blocks.read().unwrap().len() { - panic!("Unexpected block number. Expected {}, got {}", self.blocks.read().unwrap().len(), number); - } - if number > 0 { - match self.blocks.read().unwrap().get(&header.parent_hash) { - Some(parent) => { - let parent = Rlp::new(parent).val_at::(0); - if parent.number != (header.number - 1) { - panic!("Unexpected block parent"); - } - }, - None => { - panic!("Unknown block parent {:?} for block {}", header.parent_hash, number); - } - } - } - let len = self.numbers.read().unwrap().len(); - if number == len { - { - let mut difficulty = self.difficulty.write().unwrap(); - *difficulty.deref_mut() = *difficulty.deref() + header.difficulty; - } - mem::replace(self.last_hash.write().unwrap().deref_mut(), h.clone()); - self.blocks.write().unwrap().insert(h.clone(), b); - self.numbers.write().unwrap().insert(number, h.clone()); - let mut parent_hash = header.parent_hash; - if number > 0 { - let mut n = number - 1; - while n > 0 && self.numbers.read().unwrap()[&n] != parent_hash { - *self.numbers.write().unwrap().get_mut(&n).unwrap() = parent_hash.clone(); - n -= 1; - parent_hash = Rlp::new(&self.blocks.read().unwrap()[&parent_hash]).val_at::(0).parent_hash; - } - } - } - else { - self.blocks.write().unwrap().insert(h.clone(), b.to_vec()); - } - Ok(h) - } - - fn queue_info(&self) -> BlockQueueInfo { - BlockQueueInfo { - verified_queue_size: 0, - unverified_queue_size: 0, - verifying_queue_size: 0, - max_queue_size: 0, - max_mem_use: 0, - mem_used: 0, - } - } - - fn clear_queue(&self) { - } - - fn chain_info(&self) -> BlockChainInfo { - BlockChainInfo { - total_difficulty: *self.difficulty.read().unwrap(), - pending_total_difficulty: *self.difficulty.read().unwrap(), - genesis_hash: self.genesis_hash.clone(), - best_block_hash: self.last_hash.read().unwrap().clone(), - best_block_number: self.blocks.read().unwrap().len() as BlockNumber - 1, - } - } -} pub struct TestIo<'p> { pub chain: &'p mut TestBlockChainClient, From 5f37f6edb442aa41097f7c014d2a0a08b20cb1f0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 10 Mar 2016 21:01:17 +0100 Subject: [PATCH 32/33] Correct cache update order --- ethcore/src/blockchain/blockchain.rs | 52 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 8e56cdc5f..4ebd111d9 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -452,7 +452,32 @@ impl BlockChain { let batch = DBTransaction::new(); batch.put(b"best", &update.info.hash).unwrap(); - // These cached values must be updated atomically + { + let mut write_details = self.block_details.write().unwrap(); + for (hash, details) in update.block_details.into_iter() { + batch.put_extras(&hash, &details); + self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash.clone())); + write_details.insert(hash, details); + } + } + + { + let mut write_receipts = self.block_receipts.write().unwrap(); + for (hash, receipt) in &update.block_receipts { + batch.put_extras(hash, receipt); + write_receipts.remove(hash); + } + } + + { + let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); + for (bloom_hash, blocks_bloom) in &update.blocks_blooms { + batch.put_extras(bloom_hash, blocks_bloom); + write_blocks_blooms.remove(bloom_hash); + } + } + + // These cached values must be updated last and togeterh { let mut best_block = self.best_block.write().unwrap(); let mut write_hashes = self.block_hashes.write().unwrap(); @@ -481,31 +506,6 @@ impl BlockChain { } } - { - let mut write_details = self.block_details.write().unwrap(); - for (hash, details) in update.block_details.into_iter() { - batch.put_extras(&hash, &details); - self.note_used(CacheID::Extras(ExtrasIndex::BlockDetails, hash.clone())); - write_details.insert(hash, details); - } - } - - { - let mut write_receipts = self.block_receipts.write().unwrap(); - for (hash, receipt) in &update.block_receipts { - batch.put_extras(hash, receipt); - write_receipts.remove(hash); - } - } - - { - let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for (bloom_hash, blocks_bloom) in &update.blocks_blooms { - batch.put_extras(bloom_hash, blocks_bloom); - write_blocks_blooms.remove(bloom_hash); - } - } - // update extras database self.extras_db.write(batch).unwrap(); } From a6d268db1618dea7bb2c6189a443777cb0760438 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Mar 2016 10:30:13 +0100 Subject: [PATCH 33/33] fixed missing reexport --- ethcore/src/blockchain/mod.rs | 2 +- ethcore/src/client/client.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs index 6559d8364..29a4ee684 100644 --- a/ethcore/src/blockchain/mod.rs +++ b/ethcore/src/blockchain/mod.rs @@ -23,9 +23,9 @@ mod bloom_indexer; mod cache; mod tree_route; mod update; +mod import_route; #[cfg(test)] mod generator; -mod import_route; pub use self::blockchain::{BlockProvider, BlockChain, BlockChainConfig}; pub use self::cache::CacheSize; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 4ef8bb029..40f86f7a2 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -37,8 +37,9 @@ use filter::Filter; use log_entry::LocalizedLogEntry; use util::keys::store::SecretStore; use block_queue::{BlockQueue, BlockQueueInfo}; -use blockchain::{BlockChain, BlockProvider, TreeRoute, CacheSize as BlockChainCacheSize}; +use blockchain::{BlockChain, BlockProvider, TreeRoute}; use client::{BlockId, TransactionId, ClientConfig, BlockChainClient}; +pub use blockchain::CacheSize as BlockChainCacheSize; /// General block status #[derive(Debug, Eq, PartialEq)]