grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Backports for beta (#2764)

Browse Source 
* v1.3.9

* Block import optimization (#2748)

* Block import optimization

* whitespace

[ci:none]

* Don't add empty accounts to bloom (#2753)

* Incrementally calculate verification queue heap size (#2749)

* incrementally calculate queue heap size

* query the correct queue sizes
This commit is contained in:
Arkadiy Paronyan
2016-10-20 20:40:50 +02:00
committed by GitHub
parent e5ae24dfb6
commit b3097c8036
17 changed files with 258 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
ethcore/src/account.rs
View File

@@ -288,6 +288,15 @@ impl Account {
/// Determine whether there are any un-`commit()`-ed storage-setting operations.
pub fn storage_is_clean(&self) -> bool { self.storage_changes.is_empty() }
/// Check if account has zero nonce, balance, no code and no storage.
pub fn is_empty(&self) -> bool {
self.storage_changes.is_empty() &&
self.balance.is_zero() &&
self.nonce.is_zero() &&
self.storage_root == SHA3_NULL_RLP &&
self.code_hash == SHA3_EMPTY
}
#[cfg(test)]
/// return the storage root associated with this account or None if it has been altered via the overlay.
pub fn storage_root(&self) -> Option<&H256> { if self.storage_is_clean() {Some(&self.storage_root)} else {None} }

129
ethcore/src/block_queue.rs
View File

@@ -17,7 +17,7 @@
//! A queue of blocks. Sits between network or other I/O and the `BlockChain`.
//! Sorts them ready for blockchain insertion.
use std::thread::{JoinHandle, self};
use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
use std::sync::{Condvar as SCondvar, Mutex as SMutex};
use util::*;
use io::*;
@@ -76,6 +76,13 @@ impl BlockQueueInfo {
}
}
// the internal queue sizes.
struct Sizes {
unverified: AtomicUsize,
verifying: AtomicUsize,
verified: AtomicUsize,
}
/// A queue of blocks. Sits between network or other I/O and the `BlockChain`.
/// Sorts them ready for blockchain insertion.
pub struct BlockQueue {
@@ -110,7 +117,21 @@ struct QueueSignal {
impl QueueSignal {
#[cfg_attr(feature="dev", allow(bool_comparison))]
fn set(&self) {
fn set_sync(&self) {
// Do not signal when we are about to close
if self.deleting.load(AtomicOrdering::Relaxed) {
return;
}
if self.signalled.compare_and_swap(false, true, AtomicOrdering::Relaxed) == false {
if let Err(e) = self.message_channel.send_sync(ClientIoMessage::BlockVerified) {
debug!("Error sending BlockVerified message: {:?}", e);
}
}
}
#[cfg_attr(feature="dev", allow(bool_comparison))]
fn set_async(&self) {
// Do not signal when we are about to close
if self.deleting.load(AtomicOrdering::Relaxed) {
return;
@@ -131,11 +152,12 @@ impl QueueSignal {
struct Verification {
// All locks must be captured in the order declared here.
unverified: Mutex<VecDeque<UnverifiedBlock>>,
verified: Mutex<VecDeque<PreverifiedBlock>>,
verifying: Mutex<VecDeque<VerifyingBlock>>,
verified: Mutex<VecDeque<PreverifiedBlock>>,
bad: Mutex<HashSet<H256>>,
more_to_verify: SMutex<()>,
empty: SMutex<()>,
sizes: Sizes,
}
impl BlockQueue {
@@ -143,12 +165,16 @@ impl BlockQueue {
pub fn new(config: BlockQueueConfig, engine: Arc<Engine>, message_channel: IoChannel<ClientIoMessage>) -> BlockQueue {
let verification = Arc::new(Verification {
unverified: Mutex::new(VecDeque::new()),
verified: Mutex::new(VecDeque::new()),
verifying: Mutex::new(VecDeque::new()),
verified: Mutex::new(VecDeque::new()),
bad: Mutex::new(HashSet::new()),
more_to_verify: SMutex::new(()),
empty: SMutex::new(()),
sizes: Sizes {
unverified: AtomicUsize::new(0),
verifying: AtomicUsize::new(0),
verified: AtomicUsize::new(0),
}
});
let more_to_verify = Arc::new(SCondvar::new());
let deleting = Arc::new(AtomicBool::new(false));
@@ -221,16 +247,18 @@ impl BlockQueue {
}
let mut verifying = verification.verifying.lock();
let block = unverified.pop_front().unwrap();
verification.sizes.unverified.fetch_sub(block.heap_size_of_children(), AtomicOrdering::SeqCst);
verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None });
block
};
let block_hash = block.header.hash();
match verify_block_unordered(block.header, block.bytes, &*engine) {
let is_ready = match verify_block_unordered(block.header, block.bytes, &*engine) {
Ok(verified) => {
let mut verifying = verification.verifying.lock();
for e in verifying.iter_mut() {
if e.hash == block_hash {
verification.sizes.verifying.fetch_add(verified.heap_size_of_children(), AtomicOrdering::SeqCst);
e.block = Some(verified);
break;
}
@@ -239,8 +267,10 @@ impl BlockQueue {
// we're next!
let mut verified = verification.verified.lock();
let mut bad = verification.bad.lock();
BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
ready.set();
BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
true
} else {
false
}
},
Err(err) => {
@@ -250,23 +280,39 @@ impl BlockQueue {
warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err);
bad.insert(block_hash.clone());
verifying.retain(|e| e.hash != block_hash);
BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad);
ready.set();
if !verifying.is_empty() && verifying.front().unwrap().hash == block_hash {
BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad, &verification.sizes);
true
} else {
false
}
}
};
if is_ready {
// Import the block immediately
ready.set_sync();
}
}
}
fn drain_verifying(verifying: &mut VecDeque<VerifyingBlock>, verified: &mut VecDeque<PreverifiedBlock>, bad: &mut HashSet<H256>) {
fn drain_verifying(verifying: &mut VecDeque<VerifyingBlock>, verified: &mut VecDeque<PreverifiedBlock>, bad: &mut HashSet<H256>, sizes: &Sizes) {
let mut removed_size = 0;
let mut inserted_size = 0;
while !verifying.is_empty() && verifying.front().unwrap().block.is_some() {
let block = verifying.pop_front().unwrap().block.unwrap();
if bad.contains(&block.header.parent_hash) {
let size = block.heap_size_of_children();
removed_size += size;
if bad.contains(&block.header.parent_hash()) {
bad.insert(block.header.hash());
}
else {
} else {
inserted_size += size;
verified.push_back(block);
}
}
sizes.verifying.fetch_sub(removed_size, AtomicOrdering::SeqCst);
sizes.verified.fetch_add(inserted_size, AtomicOrdering::SeqCst);
}
/// Clear the queue and stop verification activity.
@@ -277,6 +323,12 @@ impl BlockQueue {
unverified.clear();
verifying.clear();
verified.clear();
let sizes = &self.verification.sizes;
sizes.unverified.store(0, AtomicOrdering::Release);
sizes.verifying.store(0, AtomicOrdering::Release);
sizes.verified.store(0, AtomicOrdering::Release);
self.processing.write().clear();
}
@@ -322,7 +374,9 @@ impl BlockQueue {
match verify_block_basic(&header, &bytes, &*self.engine) {
Ok(()) => {
self.processing.write().insert(h.clone());
self.verification.unverified.lock().push_back(UnverifiedBlock { header: header, bytes: bytes });
let block = UnverifiedBlock { header: header, bytes: bytes };
self.verification.sizes.unverified.fetch_add(block.heap_size_of_children(), AtomicOrdering::SeqCst);
self.verification.unverified.lock().push_back(block);
self.more_to_verify.notify_all();
Ok(h)
},
@@ -350,26 +404,32 @@ impl BlockQueue {
}
let mut new_verified = VecDeque::new();
let mut removed_size = 0;
for block in verified.drain(..) {
if bad.contains(&block.header.parent_hash) {
removed_size += block.heap_size_of_children();
bad.insert(block.header.hash());
processing.remove(&block.header.hash());
} else {
new_verified.push_back(block);
}
}
self.verification.sizes.verified.fetch_sub(removed_size, AtomicOrdering::SeqCst);
*verified = new_verified;
}
/// Mark given block as processed
pub fn mark_as_good(&self, block_hashes: &[H256]) {
if block_hashes.is_empty() {
return;
/// Mark given item as processed.
/// Returns true if the queue becomes empty.
pub fn mark_as_good(&self, hashes: &[H256]) -> bool {
if hashes.is_empty() {
return self.processing.read().is_empty();
}
let mut processing = self.processing.write();
for hash in block_hashes {
for hash in hashes {
processing.remove(hash);
}
processing.is_empty()
}
/// Removes up to `max` verified blocks from the queue
@@ -379,28 +439,34 @@ impl BlockQueue {
let mut result = Vec::with_capacity(count);
for _ in 0..count {
let block = verified.pop_front().unwrap();
self.verification.sizes.verified.fetch_sub(block.heap_size_of_children(), AtomicOrdering::SeqCst);
result.push(block);
}
self.ready_signal.reset();
if !verified.is_empty() {
self.ready_signal.set();
self.ready_signal.set_async();
}
result
}
/// Get queue status.
pub fn queue_info(&self) -> BlockQueueInfo {
use std::mem::size_of;
let (unverified_len, unverified_bytes) = {
let v = self.verification.unverified.lock();
(v.len(), v.heap_size_of_children())
let len = self.verification.unverified.lock().len();
let size = self.verification.sizes.unverified.load(AtomicOrdering::Acquire);
(len, size + len * size_of::<UnverifiedBlock>())
};
let (verifying_len, verifying_bytes) = {
let v = self.verification.verifying.lock();
(v.len(), v.heap_size_of_children())
let len = self.verification.verifying.lock().len();
let size = self.verification.sizes.verifying.load(AtomicOrdering::Acquire);
(len, size + len * size_of::<VerifyingBlock>())
};
let (verified_len, verified_bytes) = {
let v = self.verification.verified.lock();
(v.len(), v.heap_size_of_children())
let len = self.verification.verified.lock().len();
let size = self.verification.sizes.verified.load(AtomicOrdering::Acquire);
(len, size + len * size_of::<PreverifiedBlock>())
};
BlockQueueInfo {
unverified_queue_size: unverified_len,
@@ -408,12 +474,9 @@ impl BlockQueue {
verified_queue_size: verified_len,
max_queue_size: self.max_queue_size,
max_mem_use: self.max_mem_use,
mem_used:
unverified_bytes
+ verifying_bytes
+ verified_bytes
// TODO: https://github.com/servo/heapsize/pull/50
//+ self.processing.read().heap_size_of_children(),
mem_used: unverified_bytes
+ verifying_bytes
+ verified_bytes
}
}

25
ethcore/src/client/client.rs
View File

@@ -355,16 +355,19 @@ impl Client {
/// This is triggered by a message coming from a block queue when the block is ready for insertion
pub fn import_verified_blocks(&self) -> usize {
let max_blocks_to_import = 64;
let (imported_blocks, import_results, invalid_blocks, imported, duration) = {
let max_blocks_to_import = 4;
let (imported_blocks, import_results, invalid_blocks, imported, duration, is_empty) = {
let mut imported_blocks = Vec::with_capacity(max_blocks_to_import);
let mut invalid_blocks = HashSet::new();
let mut import_results = Vec::with_capacity(max_blocks_to_import);
let _import_lock = self.import_lock.lock();
let blocks = self.block_queue.drain(max_blocks_to_import);
if blocks.is_empty() {
return 0;
}
let _timer = PerfTimer::new("import_verified_blocks");
let start = precise_time_ns();
let blocks = self.block_queue.drain(max_blocks_to_import);
for block in blocks {
let header = &block.header;
@@ -393,23 +396,19 @@ impl Client {
let imported = imported_blocks.len();
let invalid_blocks = invalid_blocks.into_iter().collect::<Vec<H256>>();
{
if !invalid_blocks.is_empty() {
self.block_queue.mark_as_bad(&invalid_blocks);
}
if !imported_blocks.is_empty() {
self.block_queue.mark_as_good(&imported_blocks);
}
if !invalid_blocks.is_empty() {
self.block_queue.mark_as_bad(&invalid_blocks);
}
let is_empty = self.block_queue.mark_as_good(&imported_blocks);
let duration_ns = precise_time_ns() - start;
(imported_blocks, import_results, invalid_blocks, imported, duration_ns)
(imported_blocks, import_results, invalid_blocks, imported, duration_ns, is_empty)
};
{
if !imported_blocks.is_empty() && self.block_queue.queue_info().is_empty() {
if !imported_blocks.is_empty() && is_empty {
let (enacted, retracted) = self.calculate_enacted_retracted(&import_results);
if self.queue_info().is_empty() {
if is_empty {
self.miner.chain_new_blocks(self, &imported_blocks, &invalid_blocks, &enacted, &retracted);
}

1
ethcore/src/miner/miner.rs
View File

@@ -278,6 +278,7 @@ impl Miner {
trace!(target: "miner", "done recalibration.");
}
let _timer = PerfTimer::new("prepare_block");
let (transactions, mut open_block, original_work_hash) = {
let transactions = {self.transaction_queue.lock().top_transactions()};
let mut sealing_work = self.sealing_work.lock();

9
ethcore/src/snapshot/mod.rs
View File

@@ -27,7 +27,7 @@ use ids::BlockID;
use views::BlockView;
use super::state_db::StateDB;
use util::{Bytes, Hashable, HashDB, snappy, TrieDB, TrieDBMut, TrieMut, BytesConvertable};
use util::{Bytes, Hashable, HashDB, snappy, TrieDB, TrieDBMut, TrieMut, BytesConvertable, U256, Uint};
use util::Mutex;
use util::hash::{FixedHash, H256};
use util::journaldb::{self, Algorithm, JournalDB};
@@ -39,6 +39,8 @@ use self::account::Account;
use self::block::AbridgedBlock;
use self::io::SnapshotWriter;
use super::account::Account as StateAccount;
use crossbeam::{scope, ScopedJoinHandle};
use rand::{Rng, OsRng};
@@ -417,6 +419,7 @@ impl StateRebuilder {
/// Feed an uncompressed state chunk into the rebuilder.
pub fn feed(&mut self, chunk: &[u8]) -> Result<(), ::error::Error> {
let rlp = UntrustedRlp::new(chunk);
let empty_rlp = StateAccount::new_basic(U256::zero(), U256::zero()).rlp();
let account_fat_rlps: Vec<_> = rlp.iter().map(|r| r.as_raw()).collect();
let mut pairs = Vec::with_capacity(rlp.item_count());
let backing = self.db.backing().clone();
@@ -464,7 +467,9 @@ impl StateRebuilder {
};
for (hash, thin_rlp) in pairs {
bloom.set(hash.as_slice());
if &thin_rlp[..] != &empty_rlp[..] {
bloom.set(hash.as_slice());
}
try!(account_trie.insert(&hash, &thin_rlp));
}
}

25
ethcore/src/state.rs
View File

@@ -335,18 +335,20 @@ impl State {
/// Determine whether an account exists.
pub fn exists(&self, a: &Address) -> bool {
self.ensure_cached(a, RequireCache::None, |a| a.is_some())
// Bloom filter does not contain empty accounts, so it is important here to
// check if account exists in the database directly before EIP-158 is in effect.
self.ensure_cached(a, RequireCache::None, false, |a| a.is_some())
}
/// Get the balance of account `a`.
pub fn balance(&self, a: &Address) -> U256 {
self.ensure_cached(a, RequireCache::None,
self.ensure_cached(a, RequireCache::None, true,
|a| a.as_ref().map_or(U256::zero(), |account| *account.balance()))
}
/// Get the nonce of account `a`.
pub fn nonce(&self, a: &Address) -> U256 {
self.ensure_cached(a, RequireCache::None,
self.ensure_cached(a, RequireCache::None, true,
|a| a.as_ref().map_or(self.account_start_nonce, |account| *account.nonce()))
}
@@ -403,18 +405,18 @@ impl State {
/// Get accounts' code.
pub fn code(&self, a: &Address) -> Option<Arc<Bytes>> {
self.ensure_cached(a, RequireCache::Code,
self.ensure_cached(a, RequireCache::Code, true,
|a| a.as_ref().map_or(None, |a| a.code().clone()))
}
pub fn code_hash(&self, a: &Address) -> H256 {
self.ensure_cached(a, RequireCache::None,
self.ensure_cached(a, RequireCache::None, true,
|a| a.as_ref().map_or(SHA3_EMPTY, |a| a.code_hash()))
}
/// Get accounts' code size.
pub fn code_size(&self, a: &Address) -> Option<u64> {
self.ensure_cached(a, RequireCache::CodeSize,
self.ensure_cached(a, RequireCache::CodeSize, true,
|a| a.as_ref().and_then(|a| a.code_size()))
}
@@ -492,7 +494,9 @@ impl State {
for (address, ref mut a) in accounts.iter_mut().filter(|&(_, ref a)| a.is_dirty()) {
match a.account {
Some(ref mut account) => {
db.note_account_bloom(&address);
if !account.is_empty() {
db.note_account_bloom(&address);
}
let mut account_db = AccountDBMut::from_hash(db.as_hashdb_mut(), account.address_hash(address));
account.commit_storage(trie_factory, &mut account_db);
account.commit_code(&mut account_db);
@@ -545,7 +549,6 @@ impl State {
pub fn populate_from(&mut self, accounts: PodState) {
assert!(self.snapshots.borrow().is_empty());
for (add, acc) in accounts.drain().into_iter() {
self.db.note_account_bloom(&add);
self.cache.borrow_mut().insert(add, AccountEntry::new_dirty(Some(Account::from_pod(acc))));
}
}
@@ -565,7 +568,7 @@ impl State {
fn query_pod(&mut self, query: &PodState) {
for (address, pod_account) in query.get().into_iter()
.filter(|&(ref a, _)| self.ensure_cached(a, RequireCache::Code, |a| a.is_some()))
.filter(|&(ref a, _)| self.ensure_cached(a, RequireCache::Code, true, |a| a.is_some()))
{
// needs to be split into two parts for the refcell code here
// to work.
@@ -601,7 +604,7 @@ impl State {
/// Check caches for required data
/// First searches for account in the local, then the shared cache.
/// Populates local cache if nothing found.
fn ensure_cached<F, U>(&self, a: &Address, require: RequireCache, f: F) -> U
fn ensure_cached<F, U>(&self, a: &Address, require: RequireCache, check_bloom: bool, f: F) -> U
where F: Fn(Option<&Account>) -> U {
// check local cache first
if let Some(ref mut maybe_acc) = self.cache.borrow_mut().get_mut(a) {
@@ -621,6 +624,8 @@ impl State {
match result {
Some(r) => r,
None => {
// first check bloom if it is not in database for sure
if check_bloom && !self.db.check_account_bloom(a) { return f(None); }
// not found in the global cache, get from the DB and insert into local
if !self.db.check_account_bloom(a) { return f(None); }
let db = self.trie_factory.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);

2
ethcore/src/tests/client.rs
View File

@@ -152,7 +152,7 @@ fn can_handle_long_fork() {
push_blocks_to_client(client, 49, 1201, 800);
push_blocks_to_client(client, 53, 1201, 600);
for _ in 0..40 {
for _ in 0..400 {
client.import_verified_blocks();
}
assert_eq!(2000, client.chain_info().best_block_number);