grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[stable] Backports (#8133)

Browse Source 
* updater: apply exponential backoff after download failure (#8059)

* updater: apply exponential backoff after download failure

* updater: reset backoff on new release

* Limit incoming connections.  (#8060)

* Limit ingress connections
* Optimized handshakes logging

* Max code size on Kovan (#8067)

* Enable code size limit on kovan

* Fix formatting.

* add some dos protection (#8084)

* more dos protection (#8104)

* Const time comparison (#8113)

* Use `subtle::slices_equal` for constant time comparison.

Also update the existing version of subtle in `ethcrypto` from
0.1 to 0.5

* Test specifically for InvalidPassword error.

* revert removing blooms (#8066)

* Revert "fix traces, removed bloomchain crate, closes #7228, closes #7167"

This reverts commit 1bf62038678295e5586f02a38a0c5aab9a9efe62.

* Revert "fixed broken logs (#7934)"

This reverts commit f8a2e53f3e.

* fixed broken logs

* bring back old lock order

* remove migration v13

* revert CURRENT_VERSION to 12 in migration.rs

* Fix compilation.

* Check one step deeper if we're on release track branches

* add missing pr

* Fix blooms?

* Fix tests compiilation.

* Fix size.
This commit is contained in:
Tomasz Drwięga
2018-03-19 07:00:16 +01:00
committed by Marek Kotewicz
parent 3c15a501ca
commit feef2f8791
76 changed files with 2701 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ethcore/src/blockchain/block_info.rs
View File

@@ -32,7 +32,7 @@ pub struct BlockInfo {
}
/// Describes location of newly inserted block.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub enum BlockLocation {
/// It's part of the canon chain.
CanonChain,
@@ -44,7 +44,7 @@ pub enum BlockLocation {
BranchBecomingCanonChain(BranchBecomingCanonChainData),
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct BranchBecomingCanonChainData {
/// Hash of the newest common ancestor with old canon chain.
pub ancestor: H256,

206
ethcore/src/blockchain/blockchain.rs
View File

@@ -16,13 +16,15 @@
//! Blockchain database.
use std::collections::{HashMap, HashSet};
use std::collections::{HashMap, HashSet, hash_map};
use std::sync::Arc;
use std::mem;
use itertools::Itertools;
use bloomchain as bc;
use bloomchain::Bloom;
use heapsize::HeapSizeOf;
use bigint::prelude::U256;
use bigint::hash::{H256, H2048};
use bigint::hash::{H256};
use parking_lot::{Mutex, RwLock};
use bytes::Bytes;
use rlp::*;
@@ -32,6 +34,7 @@ use transaction::*;
use views::*;
use log_entry::{LogEntry, LocalizedLogEntry};
use receipt::Receipt;
use blooms::{BloomGroup, GroupPosition};
use blockchain::best_block::{BestBlock, BestAncientBlock};
use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData};
use types::blockchain_info::BlockChainInfo;
@@ -46,6 +49,9 @@ use rayon::prelude::*;
use ansi_term::Colour;
use kvdb::{DBTransaction, KeyValueDB};
const LOG_BLOOMS_LEVELS: usize = 3;
const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16;
/// Interface for querying blocks by hash and by number.
pub trait BlockProvider {
/// Returns true if the given block is known
@@ -145,7 +151,7 @@ pub trait BlockProvider {
}
/// Returns numbers of blocks containing given bloom.
fn blocks_with_blooms(&self, bloom: &[H2048], from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
/// Returns logs matching given filter.
fn logs<F>(&self, blocks: Vec<BlockNumber>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
@@ -163,14 +169,26 @@ enum CacheId {
BlockDetails(H256),
BlockHashes(BlockNumber),
TransactionAddresses(H256),
BlocksBlooms(GroupPosition),
BlockReceipts(H256),
}
impl bc::group::BloomGroupDatabase for BlockChain {
fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option<bc::group::BloomGroup> {
let position = GroupPosition::from(position.clone());
let result = self.db.read_with_cache(db::COL_EXTRA, &self.blocks_blooms, &position).map(Into::into);
self.cache_man.lock().note_used(CacheId::BlocksBlooms(position));
result
}
}
/// Structure providing fast access to blockchain data.
///
/// **Does not do input data verification.**
pub struct BlockChain {
// All locks must be captured in the order declared here.
blooms_config: bc::Config,
best_block: RwLock<BestBlock>,
// Stores best block of the first uninterrupted sequence of blocks. `None` if there are no gaps.
// Only updated with `insert_unordered_block`.
@@ -187,6 +205,7 @@ pub struct BlockChain {
block_details: RwLock<HashMap<H256, BlockDetails>>,
block_hashes: RwLock<HashMap<BlockNumber, H256>>,
transaction_addresses: RwLock<HashMap<H256, TransactionAddress>>,
blocks_blooms: RwLock<HashMap<GroupPosition, BloomGroup>>,
block_receipts: RwLock<HashMap<H256, BlockReceipts>>,
db: Arc<KeyValueDB>,
@@ -336,14 +355,13 @@ impl BlockProvider for BlockChain {
result
}
fn blocks_with_blooms(&self, blooms: &[H2048], from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
// +1, cause it's inclusive range
(from_block..to_block + 1)
.into_par_iter()
.filter_map(|number| self.block_hash(number).map(|hash| (number, hash)))
.map(|(number, hash)| (number, self.block_header_data(&hash).expect("hash exists; qed")))
.filter(|&(_, ref header)| blooms.iter().any(|bloom| header.view().log_bloom().contains(bloom)))
.map(|(number, _)| number)
/// Returns numbers of blocks containing given bloom.
fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
let range = from_block as bc::Number..to_block as bc::Number;
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.with_bloom(&range, bloom)
.into_iter()
.map(|b| b as BlockNumber)
.collect()
}
@@ -359,15 +377,19 @@ impl BlockProvider for BlockChain {
.filter_map(|number| self.block_hash(*number).map(|hash| (*number, hash)))
.filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
.filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, b.transaction_hashes())))
.flat_map(|(number, hash, receipts, hashes)| {
assert_eq!(receipts.len(), hashes.len(), "Block {} ({}) has different number of receipts ({}) to transactions ({})", number, hash, receipts.len(), hashes.len());
let mut log_index: usize = receipts.iter().map(|r| r.logs.len()).sum();
.flat_map(|(number, hash, mut receipts, mut hashes)| {
if receipts.len() != hashes.len() {
warn!("Block {} ({}) has different number of receipts ({}) to transactions ({}). Database corrupt?", number, hash, receipts.len(), hashes.len());
assert!(false);
}
let mut log_index = receipts.iter().fold(0, |sum, receipt| sum + receipt.logs.len());
let receipts_len = receipts.len();
hashes.reverse();
receipts.reverse();
receipts.into_iter()
.map(|receipt| receipt.logs)
.zip(hashes)
.rev()
.enumerate()
.flat_map(move |(index, (mut logs, tx_hash))| {
let current_log_index = log_index;
@@ -471,6 +493,10 @@ impl BlockChain {
let cache_man = CacheManager::new(config.pref_cache_size, config.max_cache_size, 400);
let mut bc = BlockChain {
blooms_config: bc::Config {
levels: LOG_BLOOMS_LEVELS,
elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX,
},
first_block: None,
best_block: RwLock::new(BestBlock::default()),
best_ancient_block: RwLock::new(None),
@@ -479,6 +505,7 @@ impl BlockChain {
block_details: RwLock::new(HashMap::new()),
block_hashes: RwLock::new(HashMap::new()),
transaction_addresses: RwLock::new(HashMap::new()),
blocks_blooms: RwLock::new(HashMap::new()),
block_receipts: RwLock::new(HashMap::new()),
db: db.clone(),
cache_man: Mutex::new(cache_man),
@@ -730,6 +757,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: self.prepare_block_details_update(bytes, &info),
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info,
timestamp: header.timestamp(),
@@ -778,6 +806,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: update,
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info,
timestamp: header.timestamp(),
@@ -925,6 +954,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: self.prepare_block_details_update(bytes, &info),
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info.clone(),
timestamp: header.timestamp(),
@@ -982,22 +1012,50 @@ impl BlockChain {
batch.extend_with_cache(db::COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove);
}
{
let mut write_blocks_blooms = self.blocks_blooms.write();
// update best block
match update.info.location {
BlockLocation::Branch => (),
BlockLocation::BranchBecomingCanonChain(_) => {
// clear all existing blooms, cause they may be created for block
// number higher than current best block
*write_blocks_blooms = update.blocks_blooms;
for (key, value) in write_blocks_blooms.iter() {
batch.write(db::COL_EXTRA, key, value);
}
},
BlockLocation::CanonChain => {
// update all existing blooms groups
for (key, value) in update.blocks_blooms {
match write_blocks_blooms.entry(key) {
hash_map::Entry::Occupied(mut entry) => {
entry.get_mut().accrue_bloom_group(&value);
batch.write(db::COL_EXTRA, entry.key(), entry.get());
},
hash_map::Entry::Vacant(entry) => {
batch.write(db::COL_EXTRA, entry.key(), &value);
entry.insert(value);
},
}
}
},
}
}
// These cached values must be updated last with all four locks taken to avoid
// cache decoherence
{
let mut best_block = self.pending_best_block.write();
match update.info.location {
BlockLocation::Branch => (),
_ => if is_best {
batch.put(db::COL_EXTRA, b"best", &update.info.hash);
*best_block = Some(BestBlock {
hash: update.info.hash,
number: update.info.number,
total_difficulty: update.info.total_difficulty,
timestamp: update.timestamp,
block: update.block.to_vec(),
});
}
if is_best && update.info.location != BlockLocation::Branch {
batch.put(db::COL_EXTRA, b"best", &update.info.hash);
*best_block = Some(BestBlock {
hash: update.info.hash,
number: update.info.number,
total_difficulty: update.info.total_difficulty,
timestamp: update.timestamp,
block: update.block.to_vec(),
});
}
let mut write_hashes = self.pending_block_hashes.write();
@@ -1216,6 +1274,59 @@ impl BlockChain {
}
}
/// This functions returns modified blocks blooms.
///
/// To accelerate blooms lookups, blomms are stored in multiple
/// layers (BLOOM_LEVELS, currently 3).
/// ChainFilter is responsible for building and rebuilding these layers.
/// It returns them in HashMap, where values are Blooms and
/// keys are BloomIndexes. BloomIndex represents bloom location on one
/// of these layers.
///
/// To reduce number of queries to databse, block blooms are stored
/// in BlocksBlooms structure which contains info about several
/// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms.
///
/// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex)
/// to bloom location in database (BlocksBloomLocation).
///
fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<GroupPosition, BloomGroup> {
let block = BlockView::new(block_bytes);
let header = block.header_view();
let log_blooms = match info.location {
BlockLocation::Branch => HashMap::new(),
BlockLocation::CanonChain => {
let log_bloom = header.log_bloom();
if log_bloom.is_zero() {
HashMap::new()
} else {
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.insert(info.number as bc::Number, log_bloom)
}
},
BlockLocation::BranchBecomingCanonChain(ref data) => {
let ancestor_number = self.block_number(&data.ancestor).unwrap();
let start_number = ancestor_number + 1;
let range = start_number as bc::Number..self.best_block_number() as bc::Number;
let mut blooms: Vec<Bloom> = data.enacted.iter()
.map(|hash| self.block_header_data(hash).unwrap())
.map(|h| h.log_bloom())
.collect();
blooms.push(header.log_bloom());
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.replace(&range, blooms)
}
};
log_blooms.into_iter()
.map(|p| (From::from(p.0), From::from(p.1)))
.collect()
}
/// Get best block hash.
pub fn best_block_hash(&self) -> H256 {
self.best_block.read().hash
@@ -1249,6 +1360,7 @@ impl BlockChain {
blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(),
block_details: self.block_details.read().heap_size_of_children(),
transaction_addresses: self.transaction_addresses.read().heap_size_of_children(),
blocks_blooms: self.blocks_blooms.read().heap_size_of_children(),
block_receipts: self.block_receipts.read().heap_size_of_children(),
}
}
@@ -1262,6 +1374,7 @@ impl BlockChain {
let mut block_details = self.block_details.write();
let mut block_hashes = self.block_hashes.write();
let mut transaction_addresses = self.transaction_addresses.write();
let mut blocks_blooms = self.blocks_blooms.write();
let mut block_receipts = self.block_receipts.write();
let mut cache_man = self.cache_man.lock();
@@ -1273,6 +1386,7 @@ impl BlockChain {
CacheId::BlockDetails(ref h) => { block_details.remove(h); }
CacheId::BlockHashes(ref h) => { block_hashes.remove(h); }
CacheId::TransactionAddresses(ref h) => { transaction_addresses.remove(h); }
CacheId::BlocksBlooms(ref h) => { blocks_blooms.remove(h); }
CacheId::BlockReceipts(ref h) => { block_receipts.remove(h); }
}
}
@@ -1282,6 +1396,7 @@ impl BlockChain {
block_details.shrink_to_fit();
block_hashes.shrink_to_fit();
transaction_addresses.shrink_to_fit();
blocks_blooms.shrink_to_fit();
block_receipts.shrink_to_fit();
block_headers.heap_size_of_children() +
@@ -1289,6 +1404,7 @@ impl BlockChain {
block_details.heap_size_of_children() +
block_hashes.heap_size_of_children() +
transaction_addresses.heap_size_of_children() +
blocks_blooms.heap_size_of_children() +
block_receipts.heap_size_of_children()
});
}
@@ -1971,46 +2087,46 @@ mod tests {
let db = new_db();
let bc = new_chain(&genesis.last().encoded(), db.clone());
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert!(blocks_b1.is_empty());
assert!(blocks_b2.is_empty());
insert_block(&db, &bc, &b1.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert!(blocks_b2.is_empty());
insert_block(&db, &bc, &b2.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
// hasn't been forked yet
insert_block(&db, &bc, &b1a.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
assert!(blocks_ba.is_empty());
// fork has happend
insert_block(&db, &bc, &b2a.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert!(blocks_b1.is_empty());
assert!(blocks_b2.is_empty());
assert_eq!(blocks_ba, vec![1, 2]);
// fork back
insert_block(&db, &bc, &b3.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
assert_eq!(blocks_ba, vec![3]);
@@ -2046,9 +2162,9 @@ mod tests {
assert_eq!(bc.block_hash(2).unwrap(), b2.last().hash());
assert_eq!(bc.block_hash(3).unwrap(), b3.last().hash());
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 3);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 3);
let blocks_b3 = bc.blocks_with_blooms(&[bloom_b3], 0, 3);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 3);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 3);
let blocks_b3 = bc.blocks_with_bloom(&bloom_b3, 0, 3);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);

4
ethcore/src/blockchain/cache.rs
View File

@@ -23,6 +23,8 @@ pub struct CacheSize {
pub block_details: usize,
/// Transaction addresses cache size.
pub transaction_addresses: usize,
/// Blooms cache size.
pub blocks_blooms: usize,
/// Block receipts size.
pub block_receipts: usize,
}
@@ -30,6 +32,6 @@ pub struct CacheSize {
impl CacheSize {
/// Total amount used by the cache.
pub fn total(&self) -> usize {
self.blocks + self.block_details + self.transaction_addresses + self.block_receipts
self.blocks + self.block_details + self.transaction_addresses + self.blocks_blooms + self.block_receipts
}
}

28
ethcore/src/blockchain/extras.rs
View File

@@ -18,6 +18,7 @@
use std::ops;
use std::io::Write;
use blooms::{GroupPosition, BloomGroup};
use db::Key;
use engines::epoch::{Transition as EpochTransition};
use header::BlockNumber;
@@ -37,6 +38,8 @@ pub enum ExtrasIndex {
BlockHash = 1,
/// Transaction address index
TransactionAddress = 2,
/// Block blooms index
BlocksBlooms = 3,
/// Block receipts index
BlockReceipts = 4,
/// Epoch transition data index.
@@ -84,6 +87,31 @@ impl Key<BlockDetails> for H256 {
}
}
pub struct LogGroupKey([u8; 6]);
impl ops::Deref for LogGroupKey {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Key<BloomGroup> for GroupPosition {
type Target = LogGroupKey;
fn key(&self) -> Self::Target {
let mut result = [0u8; 6];
result[0] = ExtrasIndex::BlocksBlooms as u8;
result[1] = self.level;
result[2] = (self.index >> 24) as u8;
result[3] = (self.index >> 16) as u8;
result[4] = (self.index >> 8) as u8;
result[5] = self.index as u8;
LogGroupKey(result)
}
}
impl Key<TransactionAddress> for H256 {
type Target = H264;

3
ethcore/src/blockchain/generator.rs
View File

@@ -17,7 +17,8 @@
//! Blockchain generator for tests.
use std::collections::VecDeque;
use bigint::prelude::{U256, H256, H2048 as Bloom};
use bigint::prelude::{U256, H256};
use bloomchain::Bloom;
use bytes::Bytes;
use header::Header;

3
ethcore/src/blockchain/update.rs
View File

@@ -3,6 +3,7 @@ use bigint::hash::H256;
use header::BlockNumber;
use blockchain::block_info::BlockInfo;
use blockchain::extras::{BlockDetails, BlockReceipts, TransactionAddress};
use blooms::{BloomGroup, GroupPosition};
/// Block extras update info.
pub struct ExtrasUpdate<'a> {
@@ -18,6 +19,8 @@ pub struct ExtrasUpdate<'a> {
pub block_details: HashMap<H256, BlockDetails>,
/// Modified block receipts.
pub block_receipts: HashMap<H256, BlockReceipts>,
/// Modified blocks blooms.
pub blocks_blooms: HashMap<GroupPosition, BloomGroup>,
/// Modified transaction addresses (None signifies removed transactions).
pub transactions_addresses: HashMap<H256, Option<TransactionAddress>>,
}

7
ethcore/src/blooms/bloom.rs
View File

@@ -28,13 +28,6 @@ impl From<LogBloom> for Bloom {
}
}
impl From<bc::Bloom> for Bloom {
fn from(bloom: bc::Bloom) -> Self {
let bytes: [u8; 256] = bloom.into();
Bloom(LogBloom::from(bytes))
}
}
impl Into<bc::Bloom> for Bloom {
fn into(self) -> bc::Bloom {
let log = self.0;

55
ethcore/src/blooms/bloom_group.rs Normal file
View File

@@ -0,0 +1,55 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use bloomchain::group as bc;
use bloomchain::{Bloom, BloomCompat};
use heapsize::HeapSizeOf;
/// Represents group of X consecutive blooms.
#[derive(Debug, Clone, RlpEncodableWrapper, RlpDecodableWrapper)]
pub struct BloomGroup {
blooms: Vec<Bloom>,
}
impl BloomGroup {
pub fn accrue_bloom_group(&mut self, group: &BloomGroup) {
for (bloom, other) in self.blooms.iter_mut().zip(group.blooms.iter()) {
bloom.accrue_bloom(other);
}
}
}
impl From<bc::BloomGroup> for BloomGroup {
fn from(group: bc::BloomGroup) -> Self {
BloomGroup {
blooms: group.blooms
}
}
}
impl Into<bc::BloomGroup> for BloomGroup {
fn into(self) -> bc::BloomGroup {
bc::BloomGroup {
blooms: self.blooms
}
}
}
impl HeapSizeOf for BloomGroup {
fn heap_size_of_children(&self) -> usize {
self.blooms.heap_size_of_children()
}
}

42
ethcore/src/blooms/group_position.rs Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2015-2017 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use bloomchain::group as bc;
use heapsize::HeapSizeOf;
/// Represents `BloomGroup` position in database.
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
pub struct GroupPosition {
/// Bloom level.
pub level: u8,
/// Group index.
pub index: u32,
}
impl From<bc::GroupPosition> for GroupPosition {
fn from(p: bc::GroupPosition) -> Self {
GroupPosition {
level: p.level as u8,
index: p.index as u32,
}
}
}
impl HeapSizeOf for GroupPosition {
fn heap_size_of_children(&self) -> usize {
0
}
}

13
ethcore/src/client/client.rs
View File

@@ -1673,8 +1673,17 @@ impl BlockChainClient for Client {
};
let chain = self.chain.read();
let blocks = chain.blocks_with_blooms(&filter.bloom_possibilities(), from, to);
chain.logs(blocks, |entry| filter.matches(entry), filter.limit)
let blocks = filter.bloom_possibilities().iter()
.map(move |bloom| {
chain.blocks_with_bloom(bloom, from, to)
})
.flat_map(|m| m)
// remove duplicate elements
.collect::<HashSet<u64>>()
.into_iter()
.collect::<Vec<u64>>();
self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit)
}
fn filter_traces(&self, filter: TraceFilter) -> Option<Vec<LocalizedTrace>> {

2
ethcore/src/lib.rs
View File

@@ -54,6 +54,7 @@
//! cargo build --release
//! ```
extern crate bloomchain;
extern crate bn;
extern crate byteorder;
extern crate crossbeam;
@@ -155,6 +156,7 @@ pub mod views;
mod cache_manager;
mod basic_types;
mod blooms;
mod pod_account;
mod state_db;
mod account_db;

10
ethcore/src/machine.rs
View File

@@ -264,15 +264,9 @@ impl EthereumMachine {
} else if block_number < ext.eip150_transition {
Schedule::new_homestead()
} else {
// There's no max_code_size transition so we tie it to eip161abc
let max_code_size = if block_number >= ext.eip161abc_transition {
self.params.max_code_size as usize
} else {
usize::max_value()
};
let max_code_size = self.params.max_code_size(block_number);
let mut schedule = Schedule::new_post_eip150(
max_code_size,
max_code_size as _,
block_number >= ext.eip160_transition,
block_number >= ext.eip161abc_transition,
block_number >= ext.eip161d_transition

14
ethcore/src/spec/spec.rs
View File

@@ -120,6 +120,8 @@ pub struct CommonParams {
pub node_permission_contract: Option<Address>,
/// Maximum contract code size that can be deployed.
pub max_code_size: u64,
/// Number of first block where max code size limit is active.
pub max_code_size_transition: BlockNumber,
/// Transaction permission managing contract address.
pub transaction_permission_contract: Option<Address>,
}
@@ -127,11 +129,20 @@ pub struct CommonParams {
impl CommonParams {
/// Schedule for an EVM in the post-EIP-150-era of the Ethereum main net.
pub fn schedule(&self, block_number: u64) -> ::vm::Schedule {
let mut schedule = ::vm::Schedule::new_post_eip150(self.max_code_size as _, true, true, true);
let mut schedule = ::vm::Schedule::new_post_eip150(self.max_code_size(block_number) as _, true, true, true);
self.update_schedule(block_number, &mut schedule);
schedule
}
/// Returns max code size at given block.
pub fn max_code_size(&self, block_number: u64) -> u64 {
if block_number >= self.max_code_size_transition {
self.max_code_size
} else {
u64::max_value()
}
}
/// Apply common spec config parameters to the schedule.
pub fn update_schedule(&self, block_number: u64, schedule: &mut ::vm::Schedule) {
schedule.have_create2 = block_number >= self.eip86_transition;
@@ -228,6 +239,7 @@ impl From<ethjson::spec::Params> for CommonParams {
registrar: p.registrar.map_or_else(Address::new, Into::into),
node_permission_contract: p.node_permission_contract.map(Into::into),
max_code_size: p.max_code_size.map_or(u64::max_value(), Into::into),
max_code_size_transition: p.max_code_size_transition.map_or(0, Into::into),
transaction_permission_contract: p.transaction_permission_contract.map(Into::into),
wasm_activation_transition: p.wasm_activation_transition.map_or(
BlockNumber::max_value(),

7
ethcore/src/trace/config.rs
View File

@@ -15,6 +15,7 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Traces config.
use bloomchain::Config as BloomConfig;
/// Traces config.
#[derive(Debug, PartialEq, Clone)]
@@ -22,6 +23,8 @@ pub struct Config {
/// Indicates if tracing should be enabled or not.
/// If it's None, it will be automatically configured.
pub enabled: bool,
/// Traces blooms configuration.
pub blooms: BloomConfig,
/// Preferef cache-size.
pub pref_cache_size: usize,
/// Max cache-size.
@@ -32,6 +35,10 @@ impl Default for Config {
fn default() -> Self {
Config {
enabled: false,
blooms: BloomConfig {
levels: 3,
elements_per_index: 16,
},
pref_cache_size: 15 * 1024 * 1024,
max_cache_size: 20 * 1024 * 1024,
}

156
ethcore/src/trace/db.rs
View File

@@ -15,15 +15,19 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Trace database.
use std::ops::Deref;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use bloomchain::{Number, Config as BloomConfig};
use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
use heapsize::HeapSizeOf;
use bigint::hash::{H256, H264, H2048 as Bloom};
use bigint::hash::{H256, H264};
use kvdb::{KeyValueDB, DBTransaction};
use parking_lot::RwLock;
use header::BlockNumber;
use trace::{LocalizedTrace, Config, Filter, Database as TraceDatabase, ImportRequest, DatabaseExtras};
use db::{self, Key, Writable, Readable, CacheUpdatePolicy};
use blooms;
use super::flat::{FlatTrace, FlatBlockTraces, FlatTransactionTraces};
use cache_manager::CacheManager;
@@ -33,8 +37,8 @@ const TRACE_DB_VER: &'static [u8] = b"1.0";
enum TraceDBIndex {
/// Block traces index.
BlockTraces = 0,
/// Blooms index.
Blooms = 2,
/// Trace bloom group index.
BloomGroups = 1,
}
impl Key<FlatBlockTraces> for H256 {
@@ -48,37 +52,80 @@ impl Key<FlatBlockTraces> for H256 {
}
}
impl Key<Bloom> for H256 {
type Target = H264;
/// Wrapper around `blooms::GroupPosition` so it could be
/// uniquely identified in the database.
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
struct TraceGroupPosition(blooms::GroupPosition);
fn key(&self) -> H264 {
let mut result = H264::default();
result[0] = TraceDBIndex::Blooms as u8;
result[1..33].copy_from_slice(self);
result
impl From<GroupPosition> for TraceGroupPosition {
fn from(position: GroupPosition) -> Self {
TraceGroupPosition(From::from(position))
}
}
impl HeapSizeOf for TraceGroupPosition {
fn heap_size_of_children(&self) -> usize {
0
}
}
/// Helper data structure created cause [u8; 6] does not implement Deref to &[u8].
pub struct TraceGroupKey([u8; 6]);
impl Deref for TraceGroupKey {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Key<blooms::BloomGroup> for TraceGroupPosition {
type Target = TraceGroupKey;
fn key(&self) -> Self::Target {
let mut result = [0u8; 6];
result[0] = TraceDBIndex::BloomGroups as u8;
result[1] = self.0.level;
result[2] = self.0.index as u8;
result[3] = (self.0.index >> 8) as u8;
result[4] = (self.0.index >> 16) as u8;
result[5] = (self.0.index >> 24) as u8;
TraceGroupKey(result)
}
}
#[derive(Debug, Hash, Eq, PartialEq)]
enum CacheId {
Trace(H256),
Bloom(H256),
Bloom(TraceGroupPosition),
}
/// Trace database.
pub struct TraceDB<T> where T: DatabaseExtras {
// cache
traces: RwLock<HashMap<H256, FlatBlockTraces>>,
blooms: RwLock<HashMap<H256, Bloom>>,
blooms: RwLock<HashMap<TraceGroupPosition, blooms::BloomGroup>>,
cache_manager: RwLock<CacheManager<CacheId>>,
// db
tracesdb: Arc<KeyValueDB>,
// config,
bloom_config: BloomConfig,
// tracing enabled
enabled: bool,
// extras
extras: Arc<T>,
}
impl<T> BloomGroupDatabase for TraceDB<T> where T: DatabaseExtras {
fn blooms_at(&self, position: &GroupPosition) -> Option<BloomGroup> {
let position = TraceGroupPosition::from(position.clone());
let result = self.tracesdb.read_with_cache(db::COL_TRACE, &self.blooms, &position).map(Into::into);
self.note_used(CacheId::Bloom(position));
result
}
}
impl<T> TraceDB<T> where T: DatabaseExtras {
/// Creates new instance of `TraceDB`.
pub fn new(config: Config, tracesdb: Arc<KeyValueDB>, extras: Arc<T>) -> Self {
@@ -90,12 +137,13 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
tracesdb.write(batch).expect("failed to update version");
TraceDB {
traces: RwLock::new(HashMap::new()),
blooms: RwLock::new(HashMap::new()),
cache_manager: RwLock::new(CacheManager::new(config.pref_cache_size, config.max_cache_size, 10 * 1024)),
tracesdb,
tracesdb: tracesdb,
bloom_config: config.blooms,
enabled: config.enabled,
extras,
traces: RwLock::default(),
blooms: RwLock::default(),
extras: extras,
}
}
@@ -140,12 +188,6 @@ impl<T> TraceDB<T> where T: DatabaseExtras {
result
}
fn bloom(&self, block_hash: &H256) -> Option<Bloom> {
let result = self.tracesdb.read_with_cache(db::COL_TRACE, &self.blooms, block_hash);
self.note_used(CacheId::Bloom(block_hash.clone()));
result
}
/// Returns vector of transaction traces for given block.
fn transactions_traces(&self, block_hash: &H256) -> Option<Vec<FlatTransactionTraces>> {
self.traces(block_hash).map(Into::into)
@@ -222,16 +264,48 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
return;
}
// now let's rebuild the blooms
if !request.enacted.is_empty() {
let range_start = request.block_number as Number + 1 - request.enacted.len();
let range_end = range_start + request.retracted;
let replaced_range = range_start..range_end;
let enacted_blooms = request.enacted
.iter()
// all traces are expected to be found here. That's why `expect` has been used
// instead of `filter_map`. If some traces haven't been found, it meens that
// traces database is corrupted or incomplete.
.map(|block_hash| if block_hash == &request.block_hash {
request.traces.bloom()
} else {
self.traces(block_hash).expect("Traces database is incomplete.").bloom()
})
.map(Into::into)
.collect();
let chain = BloomGroupChain::new(self.bloom_config, self);
let trace_blooms = chain.replace(&replaced_range, enacted_blooms);
let blooms_to_insert = trace_blooms.into_iter()
.map(|p| (From::from(p.0), From::from(p.1)))
.collect::<HashMap<TraceGroupPosition, blooms::BloomGroup>>();
let blooms_keys: Vec<_> = blooms_to_insert.keys().cloned().collect();
let mut blooms = self.blooms.write();
batch.extend_with_cache(db::COL_TRACE, &mut *blooms, blooms_to_insert, CacheUpdatePolicy::Remove);
// note_used must be called after locking blooms to avoid cache/traces deadlock on garbage collection
for key in blooms_keys {
self.note_used(CacheId::Bloom(key));
}
}
// insert new block traces into the cache and the database
let mut traces = self.traces.write();
let mut blooms = self.blooms.write();
// it's important to use overwrite here,
// cause this value might be queried by hash later
batch.write_with_cache(db::COL_TRACE, &mut *blooms, request.block_hash, request.traces.bloom(), CacheUpdatePolicy::Overwrite);
batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
self.note_used(CacheId::Trace(request.block_hash));
self.note_used(CacheId::Bloom(request.block_hash));
{
let mut traces = self.traces.write();
// it's important to use overwrite here,
// cause this value might be queried by hash later
batch.write_with_cache(db::COL_TRACE, &mut *traces, request.block_hash, request.traces, CacheUpdatePolicy::Overwrite);
// note_used must be called after locking traces to avoid cache/traces deadlock on garbage collection
self.note_used(CacheId::Trace(request.block_hash.clone()));
}
}
fn trace(&self, block_number: BlockNumber, tx_position: usize, trace_position: Vec<usize>) -> Option<LocalizedTrace> {
@@ -318,17 +392,15 @@ impl<T> TraceDatabase for TraceDB<T> where T: DatabaseExtras {
}
fn filter(&self, filter: &Filter) -> Vec<LocalizedTrace> {
let possibilities = filter.bloom_possibilities();
// + 1, cause filters are inclusive
(filter.range.start..filter.range.end + 1).into_iter()
.map(|n| n as BlockNumber)
.filter_map(|n| self.extras.block_hash(n).map(|hash| (n, hash)))
.filter(|&(_,ref hash)| {
let bloom = self.bloom(hash).expect("hash exists; qed");
possibilities.iter().any(|p| bloom.contains(p))
})
.flat_map(|(number, hash)| {
let traces = self.traces(&hash).expect("hash exists; qed");
let chain = BloomGroupChain::new(self.bloom_config, self);
let numbers = chain.filter(filter);
numbers.into_iter()
.flat_map(|n| {
let number = n as BlockNumber;
let hash = self.extras.block_hash(number)
.expect("Expected to find block hash. Extras db is probably corrupted");
let traces = self.traces(&hash)
.expect("Expected to find a trace. Db is probably corrupted.");
self.matching_block_traces(filter, traces, hash, number)
})
.collect()

21
ethcore/src/trace/types/filter.rs
View File

@@ -17,12 +17,13 @@
//! Trace filters type definitions
use std::ops::Range;
use hash::keccak;
use util::Address;
use bloomable::Bloomable;
use bigint::prelude::H2048 as Bloom;
use basic_types::LogBloom;
use bloomable::Bloomable;
use bloomchain::{Filter as BloomFilter, Number, Bloom};
use hash::keccak;
use trace::flat::FlatTrace;
use util::Address;
use super::trace::{Action, Res};
/// Addresses filter.
@@ -87,9 +88,19 @@ pub struct Filter {
pub to_address: AddressesFilter,
}
impl BloomFilter for Filter {
fn bloom_possibilities(&self) -> Vec<Bloom> {
self.bloom_possibilities()
}
fn range(&self) -> Range<Number> {
self.range.clone()
}
}
impl Filter {
/// Returns combinations of each address.
pub fn bloom_possibilities(&self) -> Vec<Bloom> {
fn bloom_possibilities(&self) -> Vec<LogBloom> {
self.to_address.with_blooms(self.from_address.blooms())
}

2
ethcore/src/verification/verification.rs
View File

@@ -460,7 +460,7 @@ mod tests {
unimplemented!()
}
fn blocks_with_blooms(&self, _blooms: &[H2048], _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber> {
fn blocks_with_bloom(&self, _bloom: &H2048, _from_block: BlockNumber, _to_block: BlockNumber) -> Vec<BlockNumber> {
unimplemented!()
}