grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[stable] Backports (#8133)

Browse Source 
* updater: apply exponential backoff after download failure (#8059)

* updater: apply exponential backoff after download failure

* updater: reset backoff on new release

* Limit incoming connections.  (#8060)

* Limit ingress connections
* Optimized handshakes logging

* Max code size on Kovan (#8067)

* Enable code size limit on kovan

* Fix formatting.

* add some dos protection (#8084)

* more dos protection (#8104)

* Const time comparison (#8113)

* Use `subtle::slices_equal` for constant time comparison.

Also update the existing version of subtle in `ethcrypto` from
0.1 to 0.5

* Test specifically for InvalidPassword error.

* revert removing blooms (#8066)

* Revert "fix traces, removed bloomchain crate, closes #7228, closes #7167"

This reverts commit 1bf62038678295e5586f02a38a0c5aab9a9efe62.

* Revert "fixed broken logs (#7934)"

This reverts commit f8a2e53f3e.

* fixed broken logs

* bring back old lock order

* remove migration v13

* revert CURRENT_VERSION to 12 in migration.rs

* Fix compilation.

* Check one step deeper if we're on release track branches

* add missing pr

* Fix blooms?

* Fix tests compiilation.

* Fix size.
This commit is contained in:
Tomasz Drwięga
2018-03-19 07:00:16 +01:00
committed by Marek Kotewicz
parent 3c15a501ca
commit feef2f8791
76 changed files with 2701 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ethcore/src/blockchain/block_info.rs
View File

@@ -32,7 +32,7 @@ pub struct BlockInfo {
}
/// Describes location of newly inserted block.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub enum BlockLocation {
/// It's part of the canon chain.
CanonChain,
@@ -44,7 +44,7 @@ pub enum BlockLocation {
BranchBecomingCanonChain(BranchBecomingCanonChainData),
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct BranchBecomingCanonChainData {
/// Hash of the newest common ancestor with old canon chain.
pub ancestor: H256,

206
ethcore/src/blockchain/blockchain.rs
View File

@@ -16,13 +16,15 @@
//! Blockchain database.
use std::collections::{HashMap, HashSet};
use std::collections::{HashMap, HashSet, hash_map};
use std::sync::Arc;
use std::mem;
use itertools::Itertools;
use bloomchain as bc;
use bloomchain::Bloom;
use heapsize::HeapSizeOf;
use bigint::prelude::U256;
use bigint::hash::{H256, H2048};
use bigint::hash::{H256};
use parking_lot::{Mutex, RwLock};
use bytes::Bytes;
use rlp::*;
@@ -32,6 +34,7 @@ use transaction::*;
use views::*;
use log_entry::{LogEntry, LocalizedLogEntry};
use receipt::Receipt;
use blooms::{BloomGroup, GroupPosition};
use blockchain::best_block::{BestBlock, BestAncientBlock};
use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData};
use types::blockchain_info::BlockChainInfo;
@@ -46,6 +49,9 @@ use rayon::prelude::*;
use ansi_term::Colour;
use kvdb::{DBTransaction, KeyValueDB};
const LOG_BLOOMS_LEVELS: usize = 3;
const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16;
/// Interface for querying blocks by hash and by number.
pub trait BlockProvider {
/// Returns true if the given block is known
@@ -145,7 +151,7 @@ pub trait BlockProvider {
}
/// Returns numbers of blocks containing given bloom.
fn blocks_with_blooms(&self, bloom: &[H2048], from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber>;
/// Returns logs matching given filter.
fn logs<F>(&self, blocks: Vec<BlockNumber>, matches: F, limit: Option<usize>) -> Vec<LocalizedLogEntry>
@@ -163,14 +169,26 @@ enum CacheId {
BlockDetails(H256),
BlockHashes(BlockNumber),
TransactionAddresses(H256),
BlocksBlooms(GroupPosition),
BlockReceipts(H256),
}
impl bc::group::BloomGroupDatabase for BlockChain {
fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option<bc::group::BloomGroup> {
let position = GroupPosition::from(position.clone());
let result = self.db.read_with_cache(db::COL_EXTRA, &self.blocks_blooms, &position).map(Into::into);
self.cache_man.lock().note_used(CacheId::BlocksBlooms(position));
result
}
}
/// Structure providing fast access to blockchain data.
///
/// **Does not do input data verification.**
pub struct BlockChain {
// All locks must be captured in the order declared here.
blooms_config: bc::Config,
best_block: RwLock<BestBlock>,
// Stores best block of the first uninterrupted sequence of blocks. `None` if there are no gaps.
// Only updated with `insert_unordered_block`.
@@ -187,6 +205,7 @@ pub struct BlockChain {
block_details: RwLock<HashMap<H256, BlockDetails>>,
block_hashes: RwLock<HashMap<BlockNumber, H256>>,
transaction_addresses: RwLock<HashMap<H256, TransactionAddress>>,
blocks_blooms: RwLock<HashMap<GroupPosition, BloomGroup>>,
block_receipts: RwLock<HashMap<H256, BlockReceipts>>,
db: Arc<KeyValueDB>,
@@ -336,14 +355,13 @@ impl BlockProvider for BlockChain {
result
}
fn blocks_with_blooms(&self, blooms: &[H2048], from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
// +1, cause it's inclusive range
(from_block..to_block + 1)
.into_par_iter()
.filter_map(|number| self.block_hash(number).map(|hash| (number, hash)))
.map(|(number, hash)| (number, self.block_header_data(&hash).expect("hash exists; qed")))
.filter(|&(_, ref header)| blooms.iter().any(|bloom| header.view().log_bloom().contains(bloom)))
.map(|(number, _)| number)
/// Returns numbers of blocks containing given bloom.
fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec<BlockNumber> {
let range = from_block as bc::Number..to_block as bc::Number;
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.with_bloom(&range, bloom)
.into_iter()
.map(|b| b as BlockNumber)
.collect()
}
@@ -359,15 +377,19 @@ impl BlockProvider for BlockChain {
.filter_map(|number| self.block_hash(*number).map(|hash| (*number, hash)))
.filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
.filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, b.transaction_hashes())))
.flat_map(|(number, hash, receipts, hashes)| {
assert_eq!(receipts.len(), hashes.len(), "Block {} ({}) has different number of receipts ({}) to transactions ({})", number, hash, receipts.len(), hashes.len());
let mut log_index: usize = receipts.iter().map(|r| r.logs.len()).sum();
.flat_map(|(number, hash, mut receipts, mut hashes)| {
if receipts.len() != hashes.len() {
warn!("Block {} ({}) has different number of receipts ({}) to transactions ({}). Database corrupt?", number, hash, receipts.len(), hashes.len());
assert!(false);
}
let mut log_index = receipts.iter().fold(0, |sum, receipt| sum + receipt.logs.len());
let receipts_len = receipts.len();
hashes.reverse();
receipts.reverse();
receipts.into_iter()
.map(|receipt| receipt.logs)
.zip(hashes)
.rev()
.enumerate()
.flat_map(move |(index, (mut logs, tx_hash))| {
let current_log_index = log_index;
@@ -471,6 +493,10 @@ impl BlockChain {
let cache_man = CacheManager::new(config.pref_cache_size, config.max_cache_size, 400);
let mut bc = BlockChain {
blooms_config: bc::Config {
levels: LOG_BLOOMS_LEVELS,
elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX,
},
first_block: None,
best_block: RwLock::new(BestBlock::default()),
best_ancient_block: RwLock::new(None),
@@ -479,6 +505,7 @@ impl BlockChain {
block_details: RwLock::new(HashMap::new()),
block_hashes: RwLock::new(HashMap::new()),
transaction_addresses: RwLock::new(HashMap::new()),
blocks_blooms: RwLock::new(HashMap::new()),
block_receipts: RwLock::new(HashMap::new()),
db: db.clone(),
cache_man: Mutex::new(cache_man),
@@ -730,6 +757,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: self.prepare_block_details_update(bytes, &info),
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info,
timestamp: header.timestamp(),
@@ -778,6 +806,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: update,
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info,
timestamp: header.timestamp(),
@@ -925,6 +954,7 @@ impl BlockChain {
block_hashes: self.prepare_block_hashes_update(bytes, &info),
block_details: self.prepare_block_details_update(bytes, &info),
block_receipts: self.prepare_block_receipts_update(receipts, &info),
blocks_blooms: self.prepare_block_blooms_update(bytes, &info),
transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info),
info: info.clone(),
timestamp: header.timestamp(),
@@ -982,22 +1012,50 @@ impl BlockChain {
batch.extend_with_cache(db::COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove);
}
{
let mut write_blocks_blooms = self.blocks_blooms.write();
// update best block
match update.info.location {
BlockLocation::Branch => (),
BlockLocation::BranchBecomingCanonChain(_) => {
// clear all existing blooms, cause they may be created for block
// number higher than current best block
*write_blocks_blooms = update.blocks_blooms;
for (key, value) in write_blocks_blooms.iter() {
batch.write(db::COL_EXTRA, key, value);
}
},
BlockLocation::CanonChain => {
// update all existing blooms groups
for (key, value) in update.blocks_blooms {
match write_blocks_blooms.entry(key) {
hash_map::Entry::Occupied(mut entry) => {
entry.get_mut().accrue_bloom_group(&value);
batch.write(db::COL_EXTRA, entry.key(), entry.get());
},
hash_map::Entry::Vacant(entry) => {
batch.write(db::COL_EXTRA, entry.key(), &value);
entry.insert(value);
},
}
}
},
}
}
// These cached values must be updated last with all four locks taken to avoid
// cache decoherence
{
let mut best_block = self.pending_best_block.write();
match update.info.location {
BlockLocation::Branch => (),
_ => if is_best {
batch.put(db::COL_EXTRA, b"best", &update.info.hash);
*best_block = Some(BestBlock {
hash: update.info.hash,
number: update.info.number,
total_difficulty: update.info.total_difficulty,
timestamp: update.timestamp,
block: update.block.to_vec(),
});
}
if is_best && update.info.location != BlockLocation::Branch {
batch.put(db::COL_EXTRA, b"best", &update.info.hash);
*best_block = Some(BestBlock {
hash: update.info.hash,
number: update.info.number,
total_difficulty: update.info.total_difficulty,
timestamp: update.timestamp,
block: update.block.to_vec(),
});
}
let mut write_hashes = self.pending_block_hashes.write();
@@ -1216,6 +1274,59 @@ impl BlockChain {
}
}
/// This functions returns modified blocks blooms.
///
/// To accelerate blooms lookups, blomms are stored in multiple
/// layers (BLOOM_LEVELS, currently 3).
/// ChainFilter is responsible for building and rebuilding these layers.
/// It returns them in HashMap, where values are Blooms and
/// keys are BloomIndexes. BloomIndex represents bloom location on one
/// of these layers.
///
/// To reduce number of queries to databse, block blooms are stored
/// in BlocksBlooms structure which contains info about several
/// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms.
///
/// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex)
/// to bloom location in database (BlocksBloomLocation).
///
fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap<GroupPosition, BloomGroup> {
let block = BlockView::new(block_bytes);
let header = block.header_view();
let log_blooms = match info.location {
BlockLocation::Branch => HashMap::new(),
BlockLocation::CanonChain => {
let log_bloom = header.log_bloom();
if log_bloom.is_zero() {
HashMap::new()
} else {
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.insert(info.number as bc::Number, log_bloom)
}
},
BlockLocation::BranchBecomingCanonChain(ref data) => {
let ancestor_number = self.block_number(&data.ancestor).unwrap();
let start_number = ancestor_number + 1;
let range = start_number as bc::Number..self.best_block_number() as bc::Number;
let mut blooms: Vec<Bloom> = data.enacted.iter()
.map(|hash| self.block_header_data(hash).unwrap())
.map(|h| h.log_bloom())
.collect();
blooms.push(header.log_bloom());
let chain = bc::group::BloomGroupChain::new(self.blooms_config, self);
chain.replace(&range, blooms)
}
};
log_blooms.into_iter()
.map(|p| (From::from(p.0), From::from(p.1)))
.collect()
}
/// Get best block hash.
pub fn best_block_hash(&self) -> H256 {
self.best_block.read().hash
@@ -1249,6 +1360,7 @@ impl BlockChain {
blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(),
block_details: self.block_details.read().heap_size_of_children(),
transaction_addresses: self.transaction_addresses.read().heap_size_of_children(),
blocks_blooms: self.blocks_blooms.read().heap_size_of_children(),
block_receipts: self.block_receipts.read().heap_size_of_children(),
}
}
@@ -1262,6 +1374,7 @@ impl BlockChain {
let mut block_details = self.block_details.write();
let mut block_hashes = self.block_hashes.write();
let mut transaction_addresses = self.transaction_addresses.write();
let mut blocks_blooms = self.blocks_blooms.write();
let mut block_receipts = self.block_receipts.write();
let mut cache_man = self.cache_man.lock();
@@ -1273,6 +1386,7 @@ impl BlockChain {
CacheId::BlockDetails(ref h) => { block_details.remove(h); }
CacheId::BlockHashes(ref h) => { block_hashes.remove(h); }
CacheId::TransactionAddresses(ref h) => { transaction_addresses.remove(h); }
CacheId::BlocksBlooms(ref h) => { blocks_blooms.remove(h); }
CacheId::BlockReceipts(ref h) => { block_receipts.remove(h); }
}
}
@@ -1282,6 +1396,7 @@ impl BlockChain {
block_details.shrink_to_fit();
block_hashes.shrink_to_fit();
transaction_addresses.shrink_to_fit();
blocks_blooms.shrink_to_fit();
block_receipts.shrink_to_fit();
block_headers.heap_size_of_children() +
@@ -1289,6 +1404,7 @@ impl BlockChain {
block_details.heap_size_of_children() +
block_hashes.heap_size_of_children() +
transaction_addresses.heap_size_of_children() +
blocks_blooms.heap_size_of_children() +
block_receipts.heap_size_of_children()
});
}
@@ -1971,46 +2087,46 @@ mod tests {
let db = new_db();
let bc = new_chain(&genesis.last().encoded(), db.clone());
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert!(blocks_b1.is_empty());
assert!(blocks_b2.is_empty());
insert_block(&db, &bc, &b1.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert!(blocks_b2.is_empty());
insert_block(&db, &bc, &b2.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
// hasn't been forked yet
insert_block(&db, &bc, &b1a.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
assert!(blocks_ba.is_empty());
// fork has happend
insert_block(&db, &bc, &b2a.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert!(blocks_b1.is_empty());
assert!(blocks_b2.is_empty());
assert_eq!(blocks_ba, vec![1, 2]);
// fork back
insert_block(&db, &bc, &b3.last().encoded(), vec![]);
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5);
let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5);
let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);
assert_eq!(blocks_ba, vec![3]);
@@ -2046,9 +2162,9 @@ mod tests {
assert_eq!(bc.block_hash(2).unwrap(), b2.last().hash());
assert_eq!(bc.block_hash(3).unwrap(), b3.last().hash());
let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 3);
let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 3);
let blocks_b3 = bc.blocks_with_blooms(&[bloom_b3], 0, 3);
let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 3);
let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 3);
let blocks_b3 = bc.blocks_with_bloom(&bloom_b3, 0, 3);
assert_eq!(blocks_b1, vec![1]);
assert_eq!(blocks_b2, vec![2]);

4
ethcore/src/blockchain/cache.rs
View File

@@ -23,6 +23,8 @@ pub struct CacheSize {
pub block_details: usize,
/// Transaction addresses cache size.
pub transaction_addresses: usize,
/// Blooms cache size.
pub blocks_blooms: usize,
/// Block receipts size.
pub block_receipts: usize,
}
@@ -30,6 +32,6 @@ pub struct CacheSize {
impl CacheSize {
/// Total amount used by the cache.
pub fn total(&self) -> usize {
self.blocks + self.block_details + self.transaction_addresses + self.block_receipts
self.blocks + self.block_details + self.transaction_addresses + self.blocks_blooms + self.block_receipts
}
}

28
ethcore/src/blockchain/extras.rs
View File

@@ -18,6 +18,7 @@
use std::ops;
use std::io::Write;
use blooms::{GroupPosition, BloomGroup};
use db::Key;
use engines::epoch::{Transition as EpochTransition};
use header::BlockNumber;
@@ -37,6 +38,8 @@ pub enum ExtrasIndex {
BlockHash = 1,
/// Transaction address index
TransactionAddress = 2,
/// Block blooms index
BlocksBlooms = 3,
/// Block receipts index
BlockReceipts = 4,
/// Epoch transition data index.
@@ -84,6 +87,31 @@ impl Key<BlockDetails> for H256 {
}
}
pub struct LogGroupKey([u8; 6]);
impl ops::Deref for LogGroupKey {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Key<BloomGroup> for GroupPosition {
type Target = LogGroupKey;
fn key(&self) -> Self::Target {
let mut result = [0u8; 6];
result[0] = ExtrasIndex::BlocksBlooms as u8;
result[1] = self.level;
result[2] = (self.index >> 24) as u8;
result[3] = (self.index >> 16) as u8;
result[4] = (self.index >> 8) as u8;
result[5] = self.index as u8;
LogGroupKey(result)
}
}
impl Key<TransactionAddress> for H256 {
type Target = H264;

3
ethcore/src/blockchain/generator.rs
View File

@@ -17,7 +17,8 @@
//! Blockchain generator for tests.
use std::collections::VecDeque;
use bigint::prelude::{U256, H256, H2048 as Bloom};
use bigint::prelude::{U256, H256};
use bloomchain::Bloom;
use bytes::Bytes;
use header::Header;

3
ethcore/src/blockchain/update.rs
View File

@@ -3,6 +3,7 @@ use bigint::hash::H256;
use header::BlockNumber;
use blockchain::block_info::BlockInfo;
use blockchain::extras::{BlockDetails, BlockReceipts, TransactionAddress};
use blooms::{BloomGroup, GroupPosition};
/// Block extras update info.
pub struct ExtrasUpdate<'a> {
@@ -18,6 +19,8 @@ pub struct ExtrasUpdate<'a> {
pub block_details: HashMap<H256, BlockDetails>,
/// Modified block receipts.
pub block_receipts: HashMap<H256, BlockReceipts>,
/// Modified blocks blooms.
pub blocks_blooms: HashMap<GroupPosition, BloomGroup>,
/// Modified transaction addresses (None signifies removed transactions).
pub transactions_addresses: HashMap<H256, Option<TransactionAddress>>,
}