From 53e8d990755125f5370a28dd630721f2498fa3e1 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 01:37:12 +0100 Subject: [PATCH 01/11] blockchain split into few separate submodules, cleanup insert_block process --- ethcore/src/blockchain/best_block.rs | 23 ++ ethcore/src/blockchain/block_info.rs | 19 + ethcore/src/{ => blockchain}/blockchain.rs | 420 ++++++++++----------- ethcore/src/blockchain/bloom_indexer.rs | 43 +++ ethcore/src/blockchain/cache.rs | 23 ++ ethcore/src/blockchain/mod.rs | 10 + ethcore/src/blockchain/tree_route.rs | 13 + ethcore/src/client.rs | 6 +- 8 files changed, 332 insertions(+), 225 deletions(-) create mode 100644 ethcore/src/blockchain/best_block.rs create mode 100644 ethcore/src/blockchain/block_info.rs rename ethcore/src/{ => blockchain}/blockchain.rs (87%) create mode 100644 ethcore/src/blockchain/bloom_indexer.rs create mode 100644 ethcore/src/blockchain/cache.rs create mode 100644 ethcore/src/blockchain/mod.rs create mode 100644 ethcore/src/blockchain/tree_route.rs diff --git a/ethcore/src/blockchain/best_block.rs b/ethcore/src/blockchain/best_block.rs new file mode 100644 index 000000000..8c7fac220 --- /dev/null +++ b/ethcore/src/blockchain/best_block.rs @@ -0,0 +1,23 @@ +use util::hash::H256; +use util::uint::U256; +use header::BlockNumber; + +/// Information about best block gathered together +#[derive(Default)] +pub struct BestBlock { + pub hash: H256, + pub number: BlockNumber, + pub total_difficulty: U256 +} + +impl BestBlock { + pub fn new() -> BestBlock { Default::default() } +} + + //BestBlock { + //hash: H256::new(), + //number: 0, + //total_difficulty: U256::from(0) + //} + //} +//} diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs new file mode 100644 index 000000000..c02fadbde --- /dev/null +++ b/ethcore/src/blockchain/block_info.rs @@ -0,0 +1,19 @@ +use util::hash::H256; +use util::uint::U256; +use header::BlockNumber; + +pub struct BlockInfo { + pub hash: H256, + pub number: BlockNumber, + pub total_difficulty: U256, + pub location: BlockLocation +} + +pub enum BlockLocation { + CanonChain, + Branch, + BranchBecomingCanonChain { + ancestor: H256, + route: Vec + } +} diff --git a/ethcore/src/blockchain.rs b/ethcore/src/blockchain/blockchain.rs similarity index 87% rename from ethcore/src/blockchain.rs rename to ethcore/src/blockchain/blockchain.rs index 7e878b81e..fa71bc817 100644 --- a/ethcore/src/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -23,113 +23,33 @@ use transaction::*; use views::*; use receipt::Receipt; use chainfilter::{ChainFilter, BloomIndex, FilterDataSource}; +use blockchain::block_info::{BlockInfo, BlockLocation}; +use blockchain::best_block::BestBlock; +use blockchain::bloom_indexer::BloomIndexer; +use blockchain::tree_route::TreeRoute; +use blockchain::CacheSize; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; -/// Represents a tree route between `from` block and `to` block: -pub struct TreeRoute { - /// A vector of hashes of all blocks, ordered from `from` to `to`. - pub blocks: Vec, - /// Best common ancestor of these blocks. - pub ancestor: H256, - /// An index where best common ancestor would be. - pub index: usize, -} - -/// Represents blockchain's in-memory cache size in bytes. -#[derive(Debug)] -pub struct CacheSize { - /// Blocks cache size. - pub blocks: usize, - /// BlockDetails cache size. - pub block_details: usize, - /// Transaction addresses cache size. - pub transaction_addresses: usize, - /// Logs cache size. - pub block_logs: usize, - /// Blooms cache size. - pub blocks_blooms: usize, - /// Block receipts size. - pub block_receipts: usize, -} - -struct BloomIndexer { - index_size: usize, - levels: u8, -} - -impl BloomIndexer { - fn new(index_size: usize, levels: u8) -> Self { - BloomIndexer { - index_size: index_size, - levels: levels - } - } - - /// Calculates bloom's position in database. - fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { - use std::{mem, ptr}; - - let hash = unsafe { - let mut hash: H256 = mem::zeroed(); - ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); - hash[8] = bloom_index.level; - hash.reverse(); - hash - }; - - BlocksBloomLocation { - hash: hash, - index: bloom_index.index % self.index_size - } - } - - fn index_size(&self) -> usize { - self.index_size - } - - fn levels(&self) -> u8 { - self.levels - } -} - /// Blockchain update info. struct ExtrasUpdate { - /// Block hash. - hash: H256, + /// Block info. + info: BlockInfo, /// DB update batch. batch: DBTransaction, - /// Inserted block familial details. - details: BlockDetails, - /// New best block (if it has changed). - new_best: Option, + /// Numbers of blocks to update in block hashes cache. + block_numbers: HashSet, + /// Hashes of blocks to update in block details cache. + block_details_hashes: HashSet, + /// Hashes of receipts to update in block receipts cache. + block_receipts_hashes: HashSet, + /// Hashes of transactions to update in transactions addresses cache. + transactions_addresses_hashes: HashSet, /// Changed blocks bloom location hashes. bloom_hashes: HashSet, } -impl CacheSize { - /// Total amount used by the cache. - fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } -} - -/// Information about best block gathered together -struct BestBlock { - pub hash: H256, - pub number: BlockNumber, - pub total_difficulty: U256 -} - -impl BestBlock { - fn new() -> BestBlock { - BestBlock { - hash: H256::new(), - number: 0, - total_difficulty: U256::from(0) - } - } -} - /// Interface for querying blocks by hash and by number. pub trait BlockProvider { /// Returns true if the given block is known @@ -452,28 +372,14 @@ impl BlockChain { /// ```json /// { blocks: [B4, B3, A3, A4], ancestor: A2, index: 2 } /// ``` - pub fn tree_route(&self, from: H256, to: H256) -> Option { - let from_details = match self.block_details(&from) { - Some(h) => h, - None => return None, - }; - let to_details = match self.block_details(&to) { - Some(h) => h, - None => return None, - }; - Some(self.tree_route_aux((&from_details, &from), (&to_details, &to))) - } - - /// Similar to `tree_route` function, but can be used to return a route - /// between blocks which may not be in database yet. - fn tree_route_aux(&self, from: (&BlockDetails, &H256), to: (&BlockDetails, &H256)) -> TreeRoute { + pub fn tree_route(&self, from: H256, to: H256) -> TreeRoute { let mut from_branch = vec![]; let mut to_branch = vec![]; - let mut from_details = from.0.clone(); - let mut to_details = to.0.clone(); - let mut current_from = from.1.clone(); - let mut current_to = to.1.clone(); + let mut from_details = self.block_details(&from).expect(&format!("Expected to find details for block {:?}", from)); + let mut to_details = self.block_details(&to).expect(&format!("Expected to find details for block {:?}", to)); + let mut current_from = from; + let mut current_to = to; // reset from && to to the same level while from_details.number > to_details.number { @@ -527,165 +433,231 @@ impl BlockChain { // store block in db self.blocks_db.put(&hash, &bytes).unwrap(); - let update = self.block_to_extras_update(bytes, receipts); - self.apply_update(update); + + let batch = DBTransaction::new(); + let info = self.block_info(bytes); + batch.put(b"best", &info.hash).unwrap(); + + self.apply_update(ExtrasUpdate { + block_numbers: self.prepare_block_hashes_update(bytes, &info, &batch), + block_details_hashes: self.prepare_block_details_update(bytes, &info, &batch), + block_receipts_hashes: self.prepare_block_receipts_update(receipts, &info, &batch), + transactions_addresses_hashes: self.prepare_transaction_addresses_update(bytes, &info, &batch), + bloom_hashes: self.prepare_block_blooms_update(bytes, &info, &batch), + batch: batch, + info: info + }); } /// Applies extras update. fn apply_update(&self, update: ExtrasUpdate) { // update best block let mut best_block = self.best_block.write().unwrap(); - if let Some(b) = update.new_best { - *best_block = b; + match update.info.location { + BlockLocation::Branch => (), + _ => { + *best_block = BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty + }; + } } - // update details cache - let mut write_details = self.block_details.write().unwrap(); - write_details.remove(&update.details.parent); - write_details.insert(update.hash.clone(), update.details); - self.note_used(CacheID::Block(update.hash)); + let mut write_hashes = self.block_hashes.write().unwrap(); + for number in &update.block_numbers { + write_hashes.remove(number); + } + + let mut write_details = self.block_details.write().unwrap(); + for hash in &update.block_details_hashes { + write_details.remove(hash); + } + + let mut write_receipts = self.block_receipts.write().unwrap(); + for hash in &update.block_receipts_hashes { + write_receipts.remove(hash); + } + + let mut write_txs = self.transaction_addresses.write().unwrap(); + for hash in &update.transactions_addresses_hashes { + write_txs.remove(hash); + } - // update blocks blooms cache let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); for bloom_hash in &update.bloom_hashes { write_blocks_blooms.remove(bloom_hash); } - // update extras database + //// update extras database self.extras_db.write(update.batch).unwrap(); } - /// Transforms block into WriteBatch that may be written into database - /// Additionally, if it's new best block it returns new best block object. - fn block_to_extras_update(&self, bytes: &[u8], receipts: Vec) -> ExtrasUpdate { - // create views onto rlp - let block = BlockView::new(bytes); + /// Get inserted block info which is critical to preapre extras updates. + fn block_info(&self, block_bytes: &[u8]) -> BlockInfo { + let block = BlockView::new(block_bytes); let header = block.header_view(); - - // prepare variables let hash = block.sha3(); - let mut parent_details = self.block_details(&header.parent_hash()).expect(format!("Invalid parent hash: {:?}", header.parent_hash()).as_ref()); + let number = header.number(); + let parent_hash = header.parent_hash(); + let parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); let total_difficulty = parent_details.total_difficulty + header.difficulty(); let is_new_best = total_difficulty > self.best_block_total_difficulty(); + + BlockInfo { + hash: hash, + number: number, + total_difficulty: total_difficulty, + location: match is_new_best { + false => BlockLocation::Branch, + true => { + // on new best block we need to make sure that all ancestors + // are moved to "canon chain" + // find the route between old best block and the new one + let best_hash = self.best_block_hash(); + let route = self.tree_route(best_hash, parent_hash); + + assert_eq!(number, parent_details.number + 1); + + match route.blocks.len() { + 0 => BlockLocation::CanonChain, + _ => BlockLocation::BranchBecomingCanonChain { + ancestor: route.ancestor, + route: route.blocks.into_iter().skip(route.index).collect() + } + } + } + } + } + } + + /// This function updates block number to block hash mappings and writes them to db batch. + /// + /// Returns modified block numbers. + fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + let block = BlockView::new(block_bytes); + let header = block.header_view(); + let number = header.number(); + + match info.location { + BlockLocation::Branch => vec![], + BlockLocation::CanonChain => { + batch.put_extras(&number, &info.hash); + vec![number] + }, + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); + let start_number = ancestor_number + 1; + + for (index, hash) in route.iter().enumerate() { + batch.put_extras(&(start_number + index as BlockNumber), hash); + } + + batch.put_extras(&number, &info.hash); + let mut numbers: Vec = (start_number..start_number + route.len() as BlockNumber).into_iter().collect(); + numbers.push(number); + numbers + } + }.into_iter().collect() + } + + /// This function creates current block details, updates parent block details and writes them to db batch. + /// + /// Returns hashes of modified block details. + fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + let block = BlockView::new(block_bytes); + let header = block.header_view(); let parent_hash = header.parent_hash(); + // update parent + let mut parent_details = self.block_details(&parent_hash).expect(format!("Invalid parent hash: {:?}", parent_hash).as_ref()); + parent_details.children.push(info.hash.clone()); + // create current block details let details = BlockDetails { number: header.number(), - total_difficulty: total_difficulty, + total_difficulty: info.total_difficulty, parent: parent_hash.clone(), children: vec![] }; - // prepare the batch - let batch = DBTransaction::new(); - - // insert new block details - batch.put_extras(&hash, &details); - - // update parent details - parent_details.children.push(hash.clone()); + // write to batch batch.put_extras(&parent_hash, &parent_details); + batch.put_extras(&info.hash, &details); + vec![parent_hash, info.hash.clone()].into_iter().collect() + } + + /// This function writes block receipts to db batch and returns hashes of modified receipts. + fn prepare_block_receipts_update(&self, receipts: Vec, info: &BlockInfo, batch: &DBTransaction) -> HashSet { + batch.put_extras(&info.hash, &BlockReceipts::new(receipts)); + vec![info.hash.clone()].into_iter().collect() + } + + /// This function writes mapping from transaction hash to transaction address to database. + /// + /// Returns hashes of modified mappings. + fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + let block = BlockView::new(block_bytes); + let transaction_hashes = block.transaction_hashes(); // update transaction addresses - for (i, tx_hash) in block.transaction_hashes().iter().enumerate() { + for (i, tx_hash) in transaction_hashes.iter().enumerate() { batch.put_extras(tx_hash, &TransactionAddress { - block_hash: hash.clone(), + block_hash: info.hash.clone(), index: i }); } - // update block receipts - batch.put_extras(&hash, &BlockReceipts::new(receipts)); + transaction_hashes.into_iter().collect() + } - // if it's not new best block, just return - if !is_new_best { - return ExtrasUpdate { - hash: hash.clone(), - batch: batch, - details: details, - new_best: None, - bloom_hashes: HashSet::new() - }; - } + /// This functions update blcoks blooms and writes them to db batch. + /// + /// TODO: this function needs comprehensive description. + fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + let block = BlockView::new(block_bytes); + let header = block.header_view(); - // if its new best block we need to make sure that all ancestors - // are moved to "canon chain" - // find the route between old best block and the new one - let best_hash = self.best_block_hash(); - let best_details = self.block_details(&best_hash).expect("best block hash is invalid!"); - let route = self.tree_route_aux((&best_details, &best_hash), (&details, &hash)); - - let modified_blooms; - - match route.blocks.len() { - // its our parent - 1 => { - batch.put_extras(&header.number(), &hash); - - // update block blooms - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .add_bloom(&header.log_bloom(), header.number() as usize); + let modified_blooms = match info.location { + BlockLocation::Branch => HashMap::new(), + BlockLocation::CanonChain => { + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .add_bloom(&header.log_bloom(), header.number() as usize) }, - // it is a fork - i if i > 1 => { - let ancestor_number = self.block_number(&route.ancestor).unwrap(); + BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { + let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - for (index, hash) in route.blocks.iter().skip(route.index).enumerate() { - batch.put_extras(&(start_number + index as BlockNumber), hash); - } - // get all blocks that are not part of canon chain (TODO: optimize it to one query) - let blooms: Vec = route.blocks.iter() - .skip(route.index) + let mut blooms: Vec = route.iter() .map(|hash| self.block(hash).unwrap()) .map(|bytes| BlockView::new(&bytes).header_view().log_bloom()) .collect(); - // reset blooms chain head - modified_blooms = ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) - .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize); - }, - // route.blocks.len() could be 0 only if inserted block is best block, - // and this is not possible at this stage - _ => { unreachable!(); } + blooms.push(header.log_bloom()); + + ChainFilter::new(self, self.bloom_indexer.index_size(), self.bloom_indexer.levels()) + .reset_chain_head(&blooms, start_number as usize, self.best_block_number() as usize) + } }; - let bloom_hashes = modified_blooms.iter() - .map(|(bloom_index, _)| self.bloom_indexer.location(&bloom_index).hash) - .collect(); - - for (bloom_hash, blocks_blooms) in modified_blooms.into_iter() + let indexed_blocks_blooms = modified_blooms.into_iter() .fold(HashMap::new(), | mut acc, (bloom_index, bloom) | { { let location = self.bloom_indexer.location(&bloom_index); let mut blocks_blooms = acc .entry(location.hash.clone()) .or_insert_with(|| self.blocks_blooms(&location.hash).unwrap_or_else(BlocksBlooms::new)); - assert_eq!(self.bloom_indexer.index_size, blocks_blooms.blooms.len()); + assert_eq!(self.bloom_indexer.index_size(), blocks_blooms.blooms.len()); blocks_blooms.blooms[location.index] = bloom; } acc - }) { - batch.put_extras(&bloom_hash, &blocks_blooms); + }); + + for (bloom_hash, blocks_blooms) in &indexed_blocks_blooms { + batch.put_extras(bloom_hash, blocks_blooms); } - // this is new best block - batch.put(b"best", &hash).unwrap(); - - let best_block = BestBlock { - hash: hash.clone(), - number: header.number(), - total_difficulty: total_difficulty - }; - - ExtrasUpdate { - hash: hash, - batch: batch, - new_best: Some(best_block), - details: details, - bloom_hashes: bloom_hashes - } + indexed_blocks_blooms.into_iter().map(|(bloom_hash, _)| bloom_hash).collect() } /// Get best block hash. @@ -886,52 +858,52 @@ mod tests { assert_eq!(bc.block_hash(3).unwrap(), b3a_hash); // test trie route - let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()).unwrap(); + let r0_1 = bc.tree_route(genesis_hash.clone(), b1_hash.clone()); assert_eq!(r0_1.ancestor, genesis_hash); assert_eq!(r0_1.blocks, [b1_hash.clone()]); assert_eq!(r0_1.index, 0); - let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()).unwrap(); + let r0_2 = bc.tree_route(genesis_hash.clone(), b2_hash.clone()); assert_eq!(r0_2.ancestor, genesis_hash); assert_eq!(r0_2.blocks, [b1_hash.clone(), b2_hash.clone()]); assert_eq!(r0_2.index, 0); - let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()).unwrap(); + let r1_3a = bc.tree_route(b1_hash.clone(), b3a_hash.clone()); assert_eq!(r1_3a.ancestor, b1_hash); assert_eq!(r1_3a.blocks, [b2_hash.clone(), b3a_hash.clone()]); assert_eq!(r1_3a.index, 0); - let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()).unwrap(); + let r1_3b = bc.tree_route(b1_hash.clone(), b3b_hash.clone()); assert_eq!(r1_3b.ancestor, b1_hash); assert_eq!(r1_3b.blocks, [b2_hash.clone(), b3b_hash.clone()]); assert_eq!(r1_3b.index, 0); - let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()).unwrap(); + let r3a_3b = bc.tree_route(b3a_hash.clone(), b3b_hash.clone()); assert_eq!(r3a_3b.ancestor, b2_hash); assert_eq!(r3a_3b.blocks, [b3a_hash.clone(), b3b_hash.clone()]); assert_eq!(r3a_3b.index, 1); - let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()).unwrap(); + let r1_0 = bc.tree_route(b1_hash.clone(), genesis_hash.clone()); assert_eq!(r1_0.ancestor, genesis_hash); assert_eq!(r1_0.blocks, [b1_hash.clone()]); assert_eq!(r1_0.index, 1); - let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()).unwrap(); + let r2_0 = bc.tree_route(b2_hash.clone(), genesis_hash.clone()); assert_eq!(r2_0.ancestor, genesis_hash); assert_eq!(r2_0.blocks, [b2_hash.clone(), b1_hash.clone()]); assert_eq!(r2_0.index, 2); - let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()).unwrap(); + let r3a_1 = bc.tree_route(b3a_hash.clone(), b1_hash.clone()); assert_eq!(r3a_1.ancestor, b1_hash); assert_eq!(r3a_1.blocks, [b3a_hash.clone(), b2_hash.clone()]); assert_eq!(r3a_1.index, 2); - let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()).unwrap(); + let r3b_1 = bc.tree_route(b3b_hash.clone(), b1_hash.clone()); assert_eq!(r3b_1.ancestor, b1_hash); assert_eq!(r3b_1.blocks, [b3b_hash.clone(), b2_hash.clone()]); assert_eq!(r3b_1.index, 2); - let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()).unwrap(); + let r3b_3a = bc.tree_route(b3b_hash.clone(), b3a_hash.clone()); assert_eq!(r3b_3a.ancestor, b2_hash); assert_eq!(r3b_3a.blocks, [b3b_hash.clone(), b3a_hash.clone()]); assert_eq!(r3b_3a.index, 1); @@ -1092,7 +1064,7 @@ mod tests { #[test] fn test_bloom_indexer() { use chainfilter::BloomIndex; - use blockchain::BloomIndexer; + use blockchain::bloom_indexer::BloomIndexer; use extras::BlocksBloomLocation; let bi = BloomIndexer::new(16, 3); diff --git a/ethcore/src/blockchain/bloom_indexer.rs b/ethcore/src/blockchain/bloom_indexer.rs new file mode 100644 index 000000000..83c15992f --- /dev/null +++ b/ethcore/src/blockchain/bloom_indexer.rs @@ -0,0 +1,43 @@ +use util::hash::H256; +use chainfilter::BloomIndex; +use extras::BlocksBloomLocation; + +pub struct BloomIndexer { + index_size: usize, + levels: u8, +} + +impl BloomIndexer { + pub fn new(index_size: usize, levels: u8) -> Self { + BloomIndexer { + index_size: index_size, + levels: levels + } + } + + /// Calculates bloom's position in database. + pub fn location(&self, bloom_index: &BloomIndex) -> BlocksBloomLocation { + use std::{mem, ptr}; + + let hash = unsafe { + let mut hash: H256 = mem::zeroed(); + ptr::copy(&[bloom_index.index / self.index_size] as *const usize as *const u8, hash.as_mut_ptr(), 8); + hash[8] = bloom_index.level; + hash.reverse(); + hash + }; + + BlocksBloomLocation { + hash: hash, + index: bloom_index.index % self.index_size + } + } + + pub fn index_size(&self) -> usize { + self.index_size + } + + pub fn levels(&self) -> u8 { + self.levels + } +} diff --git a/ethcore/src/blockchain/cache.rs b/ethcore/src/blockchain/cache.rs new file mode 100644 index 000000000..c9536bdb4 --- /dev/null +++ b/ethcore/src/blockchain/cache.rs @@ -0,0 +1,23 @@ + + +/// Represents blockchain's in-memory cache size in bytes. +#[derive(Debug)] +pub struct CacheSize { + /// Blocks cache size. + pub blocks: usize, + /// BlockDetails cache size. + pub block_details: usize, + /// Transaction addresses cache size. + pub transaction_addresses: usize, + /// Logs cache size. + pub block_logs: usize, + /// Blooms cache size. + pub blocks_blooms: usize, + /// Block receipts size. + pub block_receipts: usize, +} + +impl CacheSize { + /// Total amount used by the cache. + pub fn total(&self) -> usize { self.blocks + self.block_details + self.transaction_addresses + self.block_logs + self.blocks_blooms } +} diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs new file mode 100644 index 000000000..697c5375b --- /dev/null +++ b/ethcore/src/blockchain/mod.rs @@ -0,0 +1,10 @@ +pub mod blockchain; +mod best_block; +mod block_info; +mod bloom_indexer; +mod cache; +mod tree_route; + +pub use self::blockchain::{BlockProvider, BlockChain}; +pub use self::cache::CacheSize; +pub use self::tree_route::TreeRoute; diff --git a/ethcore/src/blockchain/tree_route.rs b/ethcore/src/blockchain/tree_route.rs new file mode 100644 index 000000000..b7a230b34 --- /dev/null +++ b/ethcore/src/blockchain/tree_route.rs @@ -0,0 +1,13 @@ +use util::hash::H256; + +/// Represents a tree route between `from` block and `to` block: +#[derive(Debug)] +pub struct TreeRoute { + /// A vector of hashes of all blocks, ordered from `from` to `to`. + pub blocks: Vec, + /// Best common ancestor of these blocks. + pub ancestor: H256, + /// An index where best common ancestor would be. + pub index: usize, +} + diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index b2ac2b9e1..33365b63c 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -408,7 +408,11 @@ impl BlockChainClient for Client { } fn tree_route(&self, from: &H256, to: &H256) -> Option { - self.chain.read().unwrap().tree_route(from.clone(), to.clone()) + let chain = self.chain.read().unwrap(); + match chain.is_known(from) && chain.is_known(to) { + true => Some(chain.tree_route(from.clone(), to.clone())), + false => None + } } fn state_data(&self, _hash: &H256) -> Option { From cd43e32e25d9809b8ceb19b0bf5b8735f3e8294a Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 02:16:39 +0100 Subject: [PATCH 02/11] added docs and license headers --- ethcore/src/blockchain/best_block.rs | 33 +++++++------ ethcore/src/blockchain/block_info.rs | 29 ++++++++++++ ethcore/src/blockchain/blockchain.rs | 58 ++++------------------- ethcore/src/blockchain/bloom_indexer.rs | 61 ++++++++++++++++++++++++- ethcore/src/blockchain/cache.rs | 14 ++++++ ethcore/src/blockchain/mod.rs | 19 ++++++++ ethcore/src/blockchain/tree_route.rs | 16 +++++++ ethcore/src/blockchain/update.rs | 23 ++++++++++ ethcore/src/extras.rs | 9 ---- 9 files changed, 189 insertions(+), 73 deletions(-) create mode 100644 ethcore/src/blockchain/update.rs diff --git a/ethcore/src/blockchain/best_block.rs b/ethcore/src/blockchain/best_block.rs index 8c7fac220..cbb219617 100644 --- a/ethcore/src/blockchain/best_block.rs +++ b/ethcore/src/blockchain/best_block.rs @@ -1,23 +1,30 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + use util::hash::H256; use util::uint::U256; use header::BlockNumber; -/// Information about best block gathered together +/// Best block info. #[derive(Default)] pub struct BestBlock { + /// Best block hash. pub hash: H256, + /// Best block number. pub number: BlockNumber, + /// Best block total difficulty. pub total_difficulty: U256 } - -impl BestBlock { - pub fn new() -> BestBlock { Default::default() } -} - - //BestBlock { - //hash: H256::new(), - //number: 0, - //total_difficulty: U256::from(0) - //} - //} -//} diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs index c02fadbde..98dac648d 100644 --- a/ethcore/src/blockchain/block_info.rs +++ b/ethcore/src/blockchain/block_info.rs @@ -1,19 +1,48 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + use util::hash::H256; use util::uint::U256; use header::BlockNumber; +/// Brief info about inserted block. pub struct BlockInfo { + /// Block hash. pub hash: H256, + /// Block number. pub number: BlockNumber, + /// Total block difficulty. pub total_difficulty: U256, + /// Block location in blockchain. pub location: BlockLocation } +/// Describes location of newly inserted block. pub enum BlockLocation { + /// It's part of the canon chain. CanonChain, + /// It's not a part of the canon chain. Branch, + /// It's part of the fork which should become canon chain, + /// because it's total difficulty is higher than current + /// canon chain difficulty. BranchBecomingCanonChain { + /// Hash of the newest common ancestor with old canon chain. ancestor: H256, + /// Hashes of the blocks between ancestor and this block. route: Vec } } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index fa71bc817..36755c891 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -27,29 +27,12 @@ use blockchain::block_info::{BlockInfo, BlockLocation}; use blockchain::best_block::BestBlock; use blockchain::bloom_indexer::BloomIndexer; use blockchain::tree_route::TreeRoute; +use blockchain::update::ExtrasUpdate; use blockchain::CacheSize; const BLOOM_INDEX_SIZE: usize = 16; const BLOOM_LEVELS: u8 = 3; -/// Blockchain update info. -struct ExtrasUpdate { - /// Block info. - info: BlockInfo, - /// DB update batch. - batch: DBTransaction, - /// Numbers of blocks to update in block hashes cache. - block_numbers: HashSet, - /// Hashes of blocks to update in block details cache. - block_details_hashes: HashSet, - /// Hashes of receipts to update in block receipts cache. - block_receipts_hashes: HashSet, - /// Hashes of transactions to update in transactions addresses cache. - transactions_addresses_hashes: HashSet, - /// Changed blocks bloom location hashes. - bloom_hashes: HashSet, -} - /// Interface for querying blocks by hash and by number. pub trait BlockProvider { /// Returns true if the given block is known @@ -271,7 +254,7 @@ impl BlockChain { let bc = BlockChain { pref_cache_size: 1 << 14, max_cache_size: 1 << 20, - best_block: RwLock::new(BestBlock::new()), + best_block: RwLock::new(BestBlock::default()), blocks: RwLock::new(HashMap::new()), block_details: RwLock::new(HashMap::new()), block_hashes: RwLock::new(HashMap::new()), @@ -376,8 +359,8 @@ impl BlockChain { let mut from_branch = vec![]; let mut to_branch = vec![]; - let mut from_details = self.block_details(&from).expect(&format!("Expected to find details for block {:?}", from)); - let mut to_details = self.block_details(&to).expect(&format!("Expected to find details for block {:?}", to)); + let mut from_details = self.block_details(&from).expect(&format!("0. Expected to find details for block {:?}", from)); + let mut to_details = self.block_details(&to).expect(&format!("1. Expected to find details for block {:?}", to)); let mut current_from = from; let mut current_to = to; @@ -385,13 +368,13 @@ impl BlockChain { while from_details.number > to_details.number { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("1. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("2. Expected to find details for block {:?}", from_details.parent)); } while to_details.number > from_details.number { to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("2. Expected to find details for block {:?}", to_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("3. Expected to find details for block {:?}", to_details.parent)); } assert_eq!(from_details.number, to_details.number); @@ -400,11 +383,11 @@ impl BlockChain { while current_from != current_to { from_branch.push(current_from); current_from = from_details.parent.clone(); - from_details = self.block_details(&from_details.parent).expect(&format!("3. Expected to find details for block {:?}", from_details.parent)); + from_details = self.block_details(&from_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); to_branch.push(current_to); current_to = to_details.parent.clone(); - to_details = self.block_details(&to_details.parent).expect(&format!("4. Expected to find details for block {:?}", from_details.parent)); + to_details = self.block_details(&to_details.parent).expect(&format!("5. Expected to find details for block {:?}", from_details.parent)); } let index = from_branch.len(); @@ -1061,30 +1044,5 @@ mod tests { assert_eq!(blocks_ba, vec![3]); } - #[test] - fn test_bloom_indexer() { - use chainfilter::BloomIndex; - use blockchain::bloom_indexer::BloomIndexer; - use extras::BlocksBloomLocation; - let bi = BloomIndexer::new(16, 3); - - let index = BloomIndex::new(0, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::new(), - index: 0 - }); - - let index = BloomIndex::new(1, 0); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), - index: 0 - }); - - let index = BloomIndex::new(0, 299_999); - assert_eq!(bi.location(&index), BlocksBloomLocation { - hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), - index: 15 - }); - } } diff --git a/ethcore/src/blockchain/bloom_indexer.rs b/ethcore/src/blockchain/bloom_indexer.rs index 83c15992f..74c679ba8 100644 --- a/ethcore/src/blockchain/bloom_indexer.rs +++ b/ethcore/src/blockchain/bloom_indexer.rs @@ -1,13 +1,39 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + use util::hash::H256; use chainfilter::BloomIndex; -use extras::BlocksBloomLocation; +/// Represents location of block bloom in extras database. +#[derive(Debug, PartialEq)] +pub struct BlocksBloomLocation { + /// Unique hash of BlocksBloom + pub hash: H256, + /// Index within BlocksBloom + pub index: usize, +} + +/// Should be used to localize blocks blooms in extras database. pub struct BloomIndexer { index_size: usize, levels: u8, } impl BloomIndexer { + /// Plain constructor. pub fn new(index_size: usize, levels: u8) -> Self { BloomIndexer { index_size: index_size, @@ -33,11 +59,44 @@ impl BloomIndexer { } } + /// Returns index size. pub fn index_size(&self) -> usize { self.index_size } + /// Returns number of cache levels. pub fn levels(&self) -> u8 { self.levels } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use util::hash::{H256, FixedHash}; + use chainfilter::BloomIndex; + use blockchain::bloom_indexer::{BloomIndexer, BlocksBloomLocation}; + + #[test] + fn test_bloom_indexer() { + let bi = BloomIndexer::new(16, 3); + + let index = BloomIndex::new(0, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::new(), + index: 0 + }); + + let index = BloomIndex::new(1, 0); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("0000000000000000000000000000000000000000000000010000000000000000").unwrap(), + index: 0 + }); + + let index = BloomIndex::new(0, 299_999); + assert_eq!(bi.location(&index), BlocksBloomLocation { + hash: H256::from_str("000000000000000000000000000000000000000000000000000000000000493d").unwrap(), + index: 15 + }); + } +} diff --git a/ethcore/src/blockchain/cache.rs b/ethcore/src/blockchain/cache.rs index c9536bdb4..722f83c16 100644 --- a/ethcore/src/blockchain/cache.rs +++ b/ethcore/src/blockchain/cache.rs @@ -1,4 +1,18 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . /// Represents blockchain's in-memory cache size in bytes. #[derive(Debug)] diff --git a/ethcore/src/blockchain/mod.rs b/ethcore/src/blockchain/mod.rs index 697c5375b..43ffb9818 100644 --- a/ethcore/src/blockchain/mod.rs +++ b/ethcore/src/blockchain/mod.rs @@ -1,9 +1,28 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Blockchain database. + pub mod blockchain; mod best_block; mod block_info; mod bloom_indexer; mod cache; mod tree_route; +mod update; pub use self::blockchain::{BlockProvider, BlockChain}; pub use self::cache::CacheSize; diff --git a/ethcore/src/blockchain/tree_route.rs b/ethcore/src/blockchain/tree_route.rs index b7a230b34..1bd0e6f75 100644 --- a/ethcore/src/blockchain/tree_route.rs +++ b/ethcore/src/blockchain/tree_route.rs @@ -1,3 +1,19 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + use util::hash::H256; /// Represents a tree route between `from` block and `to` block: diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs new file mode 100644 index 000000000..083bcbf4e --- /dev/null +++ b/ethcore/src/blockchain/update.rs @@ -0,0 +1,23 @@ +use std::collections::HashSet; +use util::hash::H256; +use util::kvdb::DBTransaction; +use header::BlockNumber; +use blockchain::block_info::BlockInfo; + +/// Block extras update info. +pub struct ExtrasUpdate { + /// Block info. + pub info: BlockInfo, + /// DB update batch. + pub batch: DBTransaction, + /// Numbers of blocks to update in block hashes cache. + pub block_numbers: HashSet, + /// Hashes of blocks to update in block details cache. + pub block_details_hashes: HashSet, + /// Hashes of receipts to update in block receipts cache. + pub block_receipts_hashes: HashSet, + /// Hashes of transactions to update in transactions addresses cache. + pub transactions_addresses_hashes: HashSet, + /// Changed blocks bloom location hashes. + pub bloom_hashes: HashSet, +} diff --git a/ethcore/src/extras.rs b/ethcore/src/extras.rs index 128357576..f4759b040 100644 --- a/ethcore/src/extras.rs +++ b/ethcore/src/extras.rs @@ -262,15 +262,6 @@ impl Encodable for BlocksBlooms { } } -/// Represents location of bloom in database. -#[derive(Debug, PartialEq)] -pub struct BlocksBloomLocation { - /// Unique hash of BlocksBloom - pub hash: H256, - /// Index within BlocksBloom - pub index: usize, -} - /// Represents address of certain transaction within block #[derive(Clone)] pub struct TransactionAddress { From 01f69ca80c85348070fffa5a1a108d83edc0d0fa Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 10:19:33 +0100 Subject: [PATCH 03/11] moved creation of blockchains db transaction to apply_update function --- ethcore/src/blockchain/blockchain.rs | 122 +++++++++++++-------------- ethcore/src/blockchain/update.rs | 16 ++-- 2 files changed, 65 insertions(+), 73 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 36755c891..6a8e1258d 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -417,23 +417,23 @@ impl BlockChain { // store block in db self.blocks_db.put(&hash, &bytes).unwrap(); - let batch = DBTransaction::new(); let info = self.block_info(bytes); - batch.put(b"best", &info.hash).unwrap(); self.apply_update(ExtrasUpdate { - block_numbers: self.prepare_block_hashes_update(bytes, &info, &batch), - block_details_hashes: self.prepare_block_details_update(bytes, &info, &batch), - block_receipts_hashes: self.prepare_block_receipts_update(receipts, &info, &batch), - transactions_addresses_hashes: self.prepare_transaction_addresses_update(bytes, &info, &batch), - bloom_hashes: self.prepare_block_blooms_update(bytes, &info, &batch), - batch: batch, + block_hashes: self.prepare_block_hashes_update(bytes, &info), + block_details: self.prepare_block_details_update(bytes, &info), + block_receipts: self.prepare_block_receipts_update(receipts, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), + blocks_blooms: self.prepare_block_blooms_update(bytes, &info), info: info }); } /// Applies extras update. fn apply_update(&self, update: ExtrasUpdate) { + let batch = DBTransaction::new(); + batch.put(b"best", &update.info.hash).unwrap(); + // update best block let mut best_block = self.best_block.write().unwrap(); match update.info.location { @@ -448,32 +448,37 @@ impl BlockChain { } let mut write_hashes = self.block_hashes.write().unwrap(); - for number in &update.block_numbers { + for (number, hash) in &update.block_hashes { + batch.put_extras(number, hash); write_hashes.remove(number); } let mut write_details = self.block_details.write().unwrap(); - for hash in &update.block_details_hashes { - write_details.remove(hash); + for (hash, details) in update.block_details.into_iter() { + batch.put_extras(&hash, &details); + write_details.insert(hash, details); } let mut write_receipts = self.block_receipts.write().unwrap(); - for hash in &update.block_receipts_hashes { + for (hash, receipt) in &update.block_receipts { + batch.put_extras(hash, receipt); write_receipts.remove(hash); } let mut write_txs = self.transaction_addresses.write().unwrap(); - for hash in &update.transactions_addresses_hashes { + for (hash, tx_address) in &update.transactions_addresses { + batch.put_extras(hash, tx_address); write_txs.remove(hash); } let mut write_blocks_blooms = self.blocks_blooms.write().unwrap(); - for bloom_hash in &update.bloom_hashes { + for (bloom_hash, blocks_bloom) in &update.blocks_blooms { + batch.put_extras(bloom_hash, blocks_bloom); write_blocks_blooms.remove(bloom_hash); } - //// update extras database - self.extras_db.write(update.batch).unwrap(); + // update extras database + self.extras_db.write(batch).unwrap(); } /// Get inserted block info which is critical to preapre extras updates. @@ -514,40 +519,35 @@ impl BlockChain { } } - /// This function updates block number to block hash mappings and writes them to db batch. - /// - /// Returns modified block numbers. - fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + /// This function returns modified block hashes. + fn prepare_block_hashes_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + let mut block_hashes = HashMap::new(); let block = BlockView::new(block_bytes); let header = block.header_view(); let number = header.number(); match info.location { - BlockLocation::Branch => vec![], + BlockLocation::Branch => (), BlockLocation::CanonChain => { - batch.put_extras(&number, &info.hash); - vec![number] + block_hashes.insert(number, info.hash.clone()); }, BlockLocation::BranchBecomingCanonChain { ref ancestor, ref route } => { let ancestor_number = self.block_number(ancestor).unwrap(); let start_number = ancestor_number + 1; - for (index, hash) in route.iter().enumerate() { - batch.put_extras(&(start_number + index as BlockNumber), hash); + for (index, hash) in route.iter().cloned().enumerate() { + block_hashes.insert(start_number + index as BlockNumber, hash); } - batch.put_extras(&number, &info.hash); - let mut numbers: Vec = (start_number..start_number + route.len() as BlockNumber).into_iter().collect(); - numbers.push(number); - numbers + block_hashes.insert(number, info.hash.clone()); } - }.into_iter().collect() + } + + block_hashes } - /// This function creates current block details, updates parent block details and writes them to db batch. - /// - /// Returns hashes of modified block details. - fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + /// This function returns modified block details. + fn prepare_block_details_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { let block = BlockView::new(block_bytes); let header = block.header_view(); let parent_hash = header.parent_hash(); @@ -565,39 +565,39 @@ impl BlockChain { }; // write to batch - batch.put_extras(&parent_hash, &parent_details); - batch.put_extras(&info.hash, &details); - vec![parent_hash, info.hash.clone()].into_iter().collect() + let mut block_details = HashMap::new(); + block_details.insert(parent_hash, parent_details); + block_details.insert(info.hash.clone(), details); + block_details } - /// This function writes block receipts to db batch and returns hashes of modified receipts. - fn prepare_block_receipts_update(&self, receipts: Vec, info: &BlockInfo, batch: &DBTransaction) -> HashSet { - batch.put_extras(&info.hash, &BlockReceipts::new(receipts)); - vec![info.hash.clone()].into_iter().collect() + /// This function returns modified block receipts. + fn prepare_block_receipts_update(&self, receipts: Vec, info: &BlockInfo) -> HashMap { + let mut block_receipts = HashMap::new(); + block_receipts.insert(info.hash.clone(), BlockReceipts::new(receipts)); + block_receipts } - /// This function writes mapping from transaction hash to transaction address to database. - /// - /// Returns hashes of modified mappings. - fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + /// This function returns modified transaction addresses. + fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { let block = BlockView::new(block_bytes); let transaction_hashes = block.transaction_hashes(); - // update transaction addresses - for (i, tx_hash) in transaction_hashes.iter().enumerate() { - batch.put_extras(tx_hash, &TransactionAddress { - block_hash: info.hash.clone(), - index: i - }); - } - - transaction_hashes.into_iter().collect() + transaction_hashes.into_iter() + .enumerate() + .fold(HashMap::new(), |mut acc, (i ,tx_hash)| { + acc.insert(tx_hash, TransactionAddress { + block_hash: info.hash.clone(), + index: i + }); + acc + }) } - /// This functions update blcoks blooms and writes them to db batch. + /// This functions returns modified blocks blooms. /// /// TODO: this function needs comprehensive description. - fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo, batch: &DBTransaction) -> HashSet { + fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { let block = BlockView::new(block_bytes); let header = block.header_view(); @@ -623,7 +623,7 @@ impl BlockChain { } }; - let indexed_blocks_blooms = modified_blooms.into_iter() + modified_blooms.into_iter() .fold(HashMap::new(), | mut acc, (bloom_index, bloom) | { { let location = self.bloom_indexer.location(&bloom_index); @@ -634,13 +634,7 @@ impl BlockChain { blocks_blooms.blooms[location.index] = bloom; } acc - }); - - for (bloom_hash, blocks_blooms) in &indexed_blocks_blooms { - batch.put_extras(bloom_hash, blocks_blooms); - } - - indexed_blocks_blooms.into_iter().map(|(bloom_hash, _)| bloom_hash).collect() + }) } /// Get best block hash. diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs index 083bcbf4e..4d0fa3881 100644 --- a/ethcore/src/blockchain/update.rs +++ b/ethcore/src/blockchain/update.rs @@ -1,23 +1,21 @@ -use std::collections::HashSet; +use std::collections::HashMap; use util::hash::H256; -use util::kvdb::DBTransaction; use header::BlockNumber; use blockchain::block_info::BlockInfo; +use extras::{BlockDetails, BlockReceipts, TransactionAddress, BlocksBlooms}; /// Block extras update info. pub struct ExtrasUpdate { /// Block info. pub info: BlockInfo, - /// DB update batch. - pub batch: DBTransaction, /// Numbers of blocks to update in block hashes cache. - pub block_numbers: HashSet, + pub block_hashes: HashMap, /// Hashes of blocks to update in block details cache. - pub block_details_hashes: HashSet, + pub block_details: HashMap, /// Hashes of receipts to update in block receipts cache. - pub block_receipts_hashes: HashSet, + pub block_receipts: HashMap, /// Hashes of transactions to update in transactions addresses cache. - pub transactions_addresses_hashes: HashSet, + pub transactions_addresses: HashMap, /// Changed blocks bloom location hashes. - pub bloom_hashes: HashSet, + pub blocks_blooms: HashMap, } From 7b3613e1f0f0fa7ba0d257b99f21436e4bda56d7 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 10:21:44 +0100 Subject: [PATCH 04/11] updated ExtrasUpdate function --- ethcore/src/blockchain/update.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs index 4d0fa3881..f8ca06e66 100644 --- a/ethcore/src/blockchain/update.rs +++ b/ethcore/src/blockchain/update.rs @@ -8,14 +8,14 @@ use extras::{BlockDetails, BlockReceipts, TransactionAddress, BlocksBlooms}; pub struct ExtrasUpdate { /// Block info. pub info: BlockInfo, - /// Numbers of blocks to update in block hashes cache. + /// Modified block hashes. pub block_hashes: HashMap, - /// Hashes of blocks to update in block details cache. + /// Modified block details. pub block_details: HashMap, - /// Hashes of receipts to update in block receipts cache. + /// Modified block receipts. pub block_receipts: HashMap, - /// Hashes of transactions to update in transactions addresses cache. + /// Modified transaction addresses. pub transactions_addresses: HashMap, - /// Changed blocks bloom location hashes. + /// Modified blocks blooms. pub blocks_blooms: HashMap, } From 1cc719d4135df70a390e132b16776d0d0f03e292 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 19:17:29 +0100 Subject: [PATCH 05/11] description for prepare_block_blooms_update function --- ethcore/src/blockchain/blockchain.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index b785629be..e09713c3b 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -588,7 +588,20 @@ impl BlockChain { /// This functions returns modified blocks blooms. /// - /// TODO: this function needs comprehensive description. + /// To accelerate blooms lookups, blomms are stored in multiple + /// layers (BLOOM_LEVELS, currently 3). + /// ChainFilter is responsible for building and rebuilding these layers. + /// It returns them in HashMap, where values are Blooms and + /// keys are BloomIndexes. BloomIndex represents bloom location on one + /// of these layers. + /// + /// To reduce number of queries to databse, block blooms are stored + /// in BlocksBlooms structure which contains info about several + /// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms. + /// + /// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex) + /// to bloom location in database (BlocksBloomLocation). + /// fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { let block = BlockView::new(block_bytes); let header = block.header_view(); From 1481f3f477e5499adea607e0264b03495fae2473 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 27 Feb 2016 19:27:34 +0100 Subject: [PATCH 06/11] replaced match with if to shorten the code --- ethcore/src/blockchain/blockchain.rs | 29 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e09713c3b..8aa0fffdc 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -488,25 +488,24 @@ impl BlockChain { hash: hash, number: number, total_difficulty: total_difficulty, - location: match is_new_best { - false => BlockLocation::Branch, - true => { - // on new best block we need to make sure that all ancestors - // are moved to "canon chain" - // find the route between old best block and the new one - let best_hash = self.best_block_hash(); - let route = self.tree_route(best_hash, parent_hash); + location: if is_new_best { + // on new best block we need to make sure that all ancestors + // are moved to "canon chain" + // find the route between old best block and the new one + let best_hash = self.best_block_hash(); + let route = self.tree_route(best_hash, parent_hash); - assert_eq!(number, parent_details.number + 1); + assert_eq!(number, parent_details.number + 1); - match route.blocks.len() { - 0 => BlockLocation::CanonChain, - _ => BlockLocation::BranchBecomingCanonChain { - ancestor: route.ancestor, - route: route.blocks.into_iter().skip(route.index).collect() - } + match route.blocks.len() { + 0 => BlockLocation::CanonChain, + _ => BlockLocation::BranchBecomingCanonChain { + ancestor: route.ancestor, + route: route.blocks.into_iter().skip(route.index).collect() } } + } else { + BlockLocation::Branch } } } From 77bfe5ae0055ee8286e663f7433028d59d874188 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 29 Feb 2016 11:58:33 +0100 Subject: [PATCH 07/11] jsonrpc uses weak pointers to client --- parity/main.rs | 6 +++--- rpc/src/v1/impls/eth.rs | 43 +++++++++++++++++++++-------------------- rpc/src/v1/impls/mod.rs | 10 ++++++++++ rpc/src/v1/impls/net.rs | 12 ++++++------ 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 6415dfc33..3d0d183a3 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -148,9 +148,9 @@ fn setup_rpc_server(client: Arc, sync: Arc, url: &str, cors_dom let mut server = rpc::HttpServer::new(1); server.add_delegate(Web3Client::new().to_delegate()); - server.add_delegate(EthClient::new(client.clone(), sync.clone()).to_delegate()); - server.add_delegate(EthFilterClient::new(client).to_delegate()); - server.add_delegate(NetClient::new(sync).to_delegate()); + server.add_delegate(EthClient::new(&client, &sync).to_delegate()); + server.add_delegate(EthFilterClient::new(&client).to_delegate()); + server.add_delegate(NetClient::new(&sync).to_delegate()); server.start_async(url, cors_domain); } diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 689afd019..00bce5437 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . //! Eth rpc implementation. -use std::sync::Arc; +use std::sync::{Arc, Weak}; use ethsync::{EthSync, SyncState}; use jsonrpc_core::*; use util::hash::*; @@ -29,21 +29,22 @@ use v1::types::{Block, BlockTransactions, BlockNumber, Bytes, SyncStatus, SyncIn /// Eth rpc implementation. pub struct EthClient { - client: Arc, - sync: Arc + client: Weak, + sync: Weak } impl EthClient { /// Creates new EthClient. - pub fn new(client: Arc, sync: Arc) -> Self { + pub fn new(client: &Arc, sync: &Arc) -> Self { EthClient { - client: client, - sync: sync + client: Arc::downgrade(client), + sync: Arc::downgrade(sync) } } fn block(&self, id: BlockId, include_txs: bool) -> Result { - match (self.client.block(id.clone()), self.client.block_total_difficulty(id)) { + let client = take_weak!(self.client); + match (client.block(id.clone()), client.block_total_difficulty(id)) { (Some(bytes), Some(total_difficulty)) => { let block_view = BlockView::new(&bytes); let view = block_view.header_view(); @@ -78,9 +79,9 @@ impl EthClient { _ => Ok(Value::Null) } } - + fn transaction(&self, id: TransactionId) -> Result { - match self.client.transaction(id) { + match take_weak!(self.client).transaction(id) { Some(t) => to_value(&Transaction::from(t)), None => Ok(Value::Null) } @@ -90,7 +91,7 @@ impl EthClient { impl Eth for EthClient { fn protocol_version(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.sync.status().protocol_version)), + Params::None => to_value(&U256::from(take_weak!(self.sync).status().protocol_version)), _ => Err(Error::invalid_params()) } } @@ -98,12 +99,12 @@ impl Eth for EthClient { fn syncing(&self, params: Params) -> Result { match params { Params::None => { - let status = self.sync.status(); + let status = take_weak!(self.sync).status(); let res = match status.state { SyncState::NotSynced | SyncState::Idle => SyncStatus::None, SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks => SyncStatus::Info(SyncInfo { starting_block: U256::from(status.start_block_number), - current_block: U256::from(self.client.chain_info().best_block_number), + current_block: U256::from(take_weak!(self.client).chain_info().best_block_number), highest_block: U256::from(status.highest_block_number.unwrap_or(status.start_block_number)) }) }; @@ -146,14 +147,14 @@ impl Eth for EthClient { fn block_number(&self, params: Params) -> Result { match params { - Params::None => to_value(&U256::from(self.client.chain_info().best_block_number)), + Params::None => to_value(&U256::from(take_weak!(self.client).chain_info().best_block_number)), _ => Err(Error::invalid_params()) } } fn block_transaction_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).transactions_count()), None => Ok(Value::Null) }) @@ -161,7 +162,7 @@ impl Eth for EthClient { fn block_uncles_count(&self, params: Params) -> Result { from_params::<(H256,)>(params) - .and_then(|(hash,)| match self.client.block(BlockId::Hash(hash)) { + .and_then(|(hash,)| match take_weak!(self.client).block(BlockId::Hash(hash)) { Some(bytes) => to_value(&BlockView::new(&bytes).uncles_count()), None => Ok(Value::Null) }) @@ -170,7 +171,7 @@ impl Eth for EthClient { // TODO: do not ignore block number param fn code_at(&self, params: Params) -> Result { from_params::<(Address, BlockNumber)>(params) - .and_then(|(address, _block_number)| to_value(&self.client.code(&address).map_or_else(Bytes::default, Bytes::new))) + .and_then(|(address, _block_number)| to_value(&take_weak!(self.client).code(&address).map_or_else(Bytes::default, Bytes::new))) } fn block_by_hash(&self, params: Params) -> Result { @@ -201,7 +202,7 @@ impl Eth for EthClient { fn logs(&self, params: Params) -> Result { from_params::<(Filter,)>(params) .and_then(|(filter,)| { - let logs = self.client.logs(filter.into()) + let logs = take_weak!(self.client).logs(filter.into()) .into_iter() .map(From::from) .collect::>(); @@ -212,14 +213,14 @@ impl Eth for EthClient { /// Eth filter rpc implementation. pub struct EthFilterClient { - client: Arc + client: Weak } impl EthFilterClient { /// Creates new Eth filter client. - pub fn new(client: Arc) -> Self { + pub fn new(client: &Arc) -> Self { EthFilterClient { - client: client + client: Arc::downgrade(client) } } } @@ -234,6 +235,6 @@ impl EthFilter for EthFilterClient { } fn filter_changes(&self, _: Params) -> Result { - to_value(&self.client.chain_info().best_block_hash).map(|v| Value::Array(vec![v])) + to_value(&take_weak!(self.client).chain_info().best_block_hash).map(|v| Value::Array(vec![v])) } } diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs index bc8b436fd..d9102b1db 100644 --- a/rpc/src/v1/impls/mod.rs +++ b/rpc/src/v1/impls/mod.rs @@ -15,6 +15,16 @@ // along with Parity. If not, see . //! Ethereum rpc interface implementation. + +macro_rules! take_weak { + ($weak: expr) => { + match $weak.upgrade() { + Some(arc) => arc, + None => return Err(Error::internal_error()) + } + } +} + mod web3; mod eth; mod net; diff --git a/rpc/src/v1/impls/net.rs b/rpc/src/v1/impls/net.rs index b36042dba..9e24caad2 100644 --- a/rpc/src/v1/impls/net.rs +++ b/rpc/src/v1/impls/net.rs @@ -15,31 +15,31 @@ // along with Parity. If not, see . //! Net rpc implementation. -use std::sync::Arc; +use std::sync::{Arc, Weak}; use jsonrpc_core::*; use ethsync::EthSync; use v1::traits::Net; /// Net rpc implementation. pub struct NetClient { - sync: Arc + sync: Weak } impl NetClient { /// Creates new NetClient. - pub fn new(sync: Arc) -> Self { + pub fn new(sync: &Arc) -> Self { NetClient { - sync: sync + sync: Arc::downgrade(sync) } } } impl Net for NetClient { fn version(&self, _: Params) -> Result { - Ok(Value::U64(self.sync.status().protocol_version as u64)) + Ok(Value::U64(take_weak!(self.sync).status().protocol_version as u64)) } fn peer_count(&self, _params: Params) -> Result { - Ok(Value::U64(self.sync.status().num_peers as u64)) + Ok(Value::U64(take_weak!(self.sync).status().num_peers as u64)) } } From 5357f581315fc6b5e179c589537b97322e6e9998 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 29 Feb 2016 12:04:58 +0100 Subject: [PATCH 08/11] uncomment state transition tests --- ethcore/src/json_tests/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index b5f28444a..f6b5751a7 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -115,7 +115,7 @@ declare_test!{StateTests_stSolidityTest, "StateTests/stSolidityTest"} declare_test!{StateTests_stSpecialTest, "StateTests/stSpecialTest"} declare_test!{StateTests_stSystemOperationsTest, "StateTests/stSystemOperationsTest"} declare_test!{StateTests_stTransactionTest, "StateTests/stTransactionTest"} -//declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} +declare_test!{StateTests_stTransitionTest, "StateTests/stTransitionTest"} declare_test!{StateTests_stWalletTest, "StateTests/stWalletTest"} From 9b9e054dc35ae8ca7953a43c8fd3a8c4aa5bfc52 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 29 Feb 2016 14:29:51 +0300 Subject: [PATCH 09/11] changing x64 asm config --- util/Cargo.toml | 2 +- util/build.rs | 10 +++++++++- util/src/lib.rs | 2 +- util/src/uint.rs | 12 ++++++------ 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/util/Cargo.toml b/util/Cargo.toml index 6a18f80de..b7eafd905 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -39,7 +39,7 @@ target_info = "0.1" [features] default = [] dev = ["clippy"] -x64asm = [] [build-dependencies] vergen = "*" +rustc_version = "0.1" diff --git a/util/build.rs b/util/build.rs index 32ee30472..20fd8c15f 100644 --- a/util/build.rs +++ b/util/build.rs @@ -1,6 +1,14 @@ extern crate vergen; +extern crate rustc_version; + + use vergen::*; +use rustc_version::{version_meta, Channel}; fn main() { vergen(OutputFns::all()).unwrap(); -} \ No newline at end of file + + if let Channel::Nightly = version_meta().channel { + println!("cargo:rustc-cfg=x64asm"); + } +} diff --git a/util/src/lib.rs b/util/src/lib.rs index be1e788c9..2a47eb438 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,7 +16,7 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] -#![cfg_attr(feature="x64asm", feature(asm))] +#![cfg_attr(x64asm, feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings diff --git a/util/src/uint.rs b/util/src/uint.rs index 67a6f2ff4..517b7a29f 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,7 +51,7 @@ macro_rules! impl_map_from { } } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(x64asm, target_arch="x86_64")))] macro_rules! uint_overflowing_add { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) @@ -89,7 +89,7 @@ macro_rules! uint_overflowing_add_reg { } -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(x64asm, target_arch="x86_64"))] macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -165,7 +165,7 @@ macro_rules! uint_overflowing_add { ) } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(x64asm, target_arch="x86_64")))] macro_rules! uint_overflowing_sub { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let res = overflowing!((!$other).overflowing_add(From::from(1u64))); @@ -174,7 +174,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(x64asm, target_arch="x86_64"))] macro_rules! uint_overflowing_sub { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -250,7 +250,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch="x86_64"))] +#[cfg(all(x64asm, target_arch="x86_64"))] macro_rules! uint_overflowing_mul { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -370,7 +370,7 @@ macro_rules! uint_overflowing_mul { ) } -#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] +#[cfg(not(all(x64asm, target_arch="x86_64")))] macro_rules! uint_overflowing_mul { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) From e0c158c12f53088fbeefd75707c9b2ffabf6a4e9 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 29 Feb 2016 14:40:59 +0300 Subject: [PATCH 10/11] removed space --- util/build.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/util/build.rs b/util/build.rs index 20fd8c15f..b8346c97d 100644 --- a/util/build.rs +++ b/util/build.rs @@ -1,7 +1,6 @@ extern crate vergen; extern crate rustc_version; - use vergen::*; use rustc_version::{version_meta, Channel}; From 06623333d9de984c22eaa6adf0e5eed2d168070f Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 29 Feb 2016 15:23:43 +0300 Subject: [PATCH 11/11] fix tabs --- util/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/build.rs b/util/build.rs index b8346c97d..2b581642b 100644 --- a/util/build.rs +++ b/util/build.rs @@ -7,7 +7,7 @@ use rustc_version::{version_meta, Channel}; fn main() { vergen(OutputFns::all()).unwrap(); - if let Channel::Nightly = version_meta().channel { + if let Channel::Nightly = version_meta().channel { println!("cargo:rustc-cfg=x64asm"); - } + } }