Extract blockchain from ethcore (#10114)

* Split blockchain & db from ethcore.

* Clean up blockchain deps.

* Missing docs.

* Fix blockchain tests.

* Make other crates compile.

* Remove some re-exports.

* Remove types re-export from ethcore.

* Remove EVM dependency from transaction.

* Merge ethcore-transaction with common-types.

* Clean-up ethcore deps a bit.

* remove ethcore from cargo.toml

* Update ethcore/blockchain/src/lib.rs

Co-Authored-By: tomusdrw <tomusdrw@users.noreply.github.com>

* Address review comments.

* Update DB comment.

* Add tracking issue to the TODO and fix typo.

* Common naming for common types.

* Update ethcore/db/src/keys.rs

Co-Authored-By: tomusdrw <tomusdrw@users.noreply.github.com>

* Update ethcore/blockchain/src/generator.rs

Co-Authored-By: tomusdrw <tomusdrw@users.noreply.github.com>

* Try to fix beta tests.
This commit is contained in:
Tomasz Drwięga
2019-01-04 14:05:46 +01:00
committed by Afri Schoedon
parent 3090324366
commit 3650f2d51c
223 changed files with 1428 additions and 1058 deletions

View File

@@ -0,0 +1,82 @@
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Database cache manager
use std::collections::{VecDeque, HashSet};
use std::hash::Hash;
const COLLECTION_QUEUE_SIZE: usize = 8;
/// DB cache manager
pub struct CacheManager<T> {
pref_cache_size: usize,
max_cache_size: usize,
bytes_per_cache_entry: usize,
cache_usage: VecDeque<HashSet<T>>
}
impl<T> CacheManager<T> where T: Eq + Hash {
/// Create new cache manager with preferred (heap) sizes.
pub fn new(pref_cache_size: usize, max_cache_size: usize, bytes_per_cache_entry: usize) -> Self {
CacheManager {
pref_cache_size: pref_cache_size,
max_cache_size: max_cache_size,
bytes_per_cache_entry: bytes_per_cache_entry,
cache_usage: (0..COLLECTION_QUEUE_SIZE).into_iter().map(|_| Default::default()).collect(),
}
}
/// Mark element as used.
pub fn note_used(&mut self, id: T) {
if !self.cache_usage[0].contains(&id) {
if let Some(c) = self.cache_usage.iter_mut().skip(1).find(|e| e.contains(&id)) {
c.remove(&id);
}
self.cache_usage[0].insert(id);
}
}
/// Collects unused objects from cache.
/// First params is the current size of the cache.
/// Second one is an with objects to remove. It should also return new size of the cache.
pub fn collect_garbage<F>(&mut self, current_size: usize, mut notify_unused: F) where F: FnMut(HashSet<T>) -> usize {
if current_size < self.pref_cache_size {
self.rotate_cache_if_needed();
return;
}
for _ in 0..COLLECTION_QUEUE_SIZE {
if let Some(back) = self.cache_usage.pop_back() {
let current_size = notify_unused(back);
self.cache_usage.push_front(Default::default());
if current_size < self.max_cache_size {
break
}
}
}
}
fn rotate_cache_if_needed(&mut self) {
if self.cache_usage.is_empty() { return }
if self.cache_usage[0].len() * self.bytes_per_cache_entry > self.pref_cache_size / COLLECTION_QUEUE_SIZE {
if let Some(cache) = self.cache_usage.pop_back() {
self.cache_usage.push_front(cache);
}
}
}
}

239
ethcore/db/src/db.rs Normal file
View File

@@ -0,0 +1,239 @@
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Database utilities and definitions.
use std::ops::Deref;
use std::hash::Hash;
use std::collections::HashMap;
use parking_lot::RwLock;
use kvdb::{DBTransaction, KeyValueDB};
use rlp;
// database columns
/// Column for State
pub const COL_STATE: Option<u32> = Some(0);
/// Column for Block headers
pub const COL_HEADERS: Option<u32> = Some(1);
/// Column for Block bodies
pub const COL_BODIES: Option<u32> = Some(2);
/// Column for Extras
pub const COL_EXTRA: Option<u32> = Some(3);
/// Column for Traces
pub const COL_TRACE: Option<u32> = Some(4);
/// Column for the empty accounts bloom filter.
pub const COL_ACCOUNT_BLOOM: Option<u32> = Some(5);
/// Column for general information from the local node which can persist.
pub const COL_NODE_INFO: Option<u32> = Some(6);
/// Column for the light client chain.
pub const COL_LIGHT_CHAIN: Option<u32> = Some(7);
/// Number of columns in DB
pub const NUM_COLUMNS: Option<u32> = Some(8);
/// Modes for updating caches.
#[derive(Clone, Copy)]
pub enum CacheUpdatePolicy {
/// Overwrite entries.
Overwrite,
/// Remove entries.
Remove,
}
/// A cache for arbitrary key-value pairs.
pub trait Cache<K, V> {
/// Insert an entry into the cache and get the old value.
fn insert(&mut self, k: K, v: V) -> Option<V>;
/// Remove an entry from the cache, getting the old value if it existed.
fn remove(&mut self, k: &K) -> Option<V>;
/// Query the cache for a key's associated value.
fn get(&self, k: &K) -> Option<&V>;
}
impl<K, V> Cache<K, V> for HashMap<K, V> where K: Hash + Eq {
fn insert(&mut self, k: K, v: V) -> Option<V> {
HashMap::insert(self, k, v)
}
fn remove(&mut self, k: &K) -> Option<V> {
HashMap::remove(self, k)
}
fn get(&self, k: &K) -> Option<&V> {
HashMap::get(self, k)
}
}
/// Should be used to get database key associated with given value.
pub trait Key<T> {
/// The db key associated with this value.
type Target: Deref<Target = [u8]>;
/// Returns db key.
fn key(&self) -> Self::Target;
}
/// Should be used to write value into database.
pub trait Writable {
/// Writes the value into the database.
fn write<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>, value: &T) where T: rlp::Encodable, R: Deref<Target = [u8]>;
/// Deletes key from the databse.
fn delete<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>) where T: rlp::Encodable, R: Deref<Target = [u8]>;
/// Writes the value into the database and updates the cache.
fn write_with_cache<K, T, R>(&mut self, col: Option<u32>, cache: &mut Cache<K, T>, key: K, value: T, policy: CacheUpdatePolicy) where
K: Key<T, Target = R> + Hash + Eq,
T: rlp::Encodable,
R: Deref<Target = [u8]> {
self.write(col, &key, &value);
match policy {
CacheUpdatePolicy::Overwrite => {
cache.insert(key, value);
},
CacheUpdatePolicy::Remove => {
cache.remove(&key);
}
}
}
/// Writes the values into the database and updates the cache.
fn extend_with_cache<K, T, R>(&mut self, col: Option<u32>, cache: &mut Cache<K, T>, values: HashMap<K, T>, policy: CacheUpdatePolicy) where
K: Key<T, Target = R> + Hash + Eq,
T: rlp::Encodable,
R: Deref<Target = [u8]> {
match policy {
CacheUpdatePolicy::Overwrite => {
for (key, value) in values {
self.write(col, &key, &value);
cache.insert(key, value);
}
},
CacheUpdatePolicy::Remove => {
for (key, value) in &values {
self.write(col, key, value);
cache.remove(key);
}
},
}
}
/// Writes and removes the values into the database and updates the cache.
fn extend_with_option_cache<K, T, R>(&mut self, col: Option<u32>, cache: &mut Cache<K, Option<T>>, values: HashMap<K, Option<T>>, policy: CacheUpdatePolicy) where
K: Key<T, Target = R> + Hash + Eq,
T: rlp::Encodable,
R: Deref<Target = [u8]> {
match policy {
CacheUpdatePolicy::Overwrite => {
for (key, value) in values {
match value {
Some(ref v) => self.write(col, &key, v),
None => self.delete(col, &key),
}
cache.insert(key, value);
}
},
CacheUpdatePolicy::Remove => {
for (key, value) in values {
match value {
Some(v) => self.write(col, &key, &v),
None => self.delete(col, &key),
}
cache.remove(&key);
}
},
}
}
}
/// Should be used to read values from database.
pub trait Readable {
/// Returns value for given key.
fn read<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> Option<T> where
T: rlp::Decodable,
R: Deref<Target = [u8]>;
/// Returns value for given key either in cache or in database.
fn read_with_cache<K, T, C>(&self, col: Option<u32>, cache: &RwLock<C>, key: &K) -> Option<T> where
K: Key<T> + Eq + Hash + Clone,
T: Clone + rlp::Decodable,
C: Cache<K, T> {
{
let read = cache.read();
if let Some(v) = read.get(key) {
return Some(v.clone());
}
}
self.read(col, key).map(|value: T|{
let mut write = cache.write();
write.insert(key.clone(), value.clone());
value
})
}
/// Returns true if given value exists.
fn exists<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> bool where R: Deref<Target= [u8]>;
/// Returns true if given value exists either in cache or in database.
fn exists_with_cache<K, T, R, C>(&self, col: Option<u32>, cache: &RwLock<C>, key: &K) -> bool where
K: Eq + Hash + Key<T, Target = R>,
R: Deref<Target = [u8]>,
C: Cache<K, T> {
{
let read = cache.read();
if read.get(key).is_some() {
return true;
}
}
self.exists::<T, R>(col, key)
}
}
impl Writable for DBTransaction {
fn write<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>, value: &T) where T: rlp::Encodable, R: Deref<Target = [u8]> {
self.put(col, &key.key(), &rlp::encode(value));
}
fn delete<T, R>(&mut self, col: Option<u32>, key: &Key<T, Target = R>) where T: rlp::Encodable, R: Deref<Target = [u8]> {
self.delete(col, &key.key());
}
}
impl<KVDB: KeyValueDB + ?Sized> Readable for KVDB {
fn read<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> Option<T>
where T: rlp::Decodable, R: Deref<Target = [u8]> {
self.get(col, &key.key())
.expect(&format!("db get failed, key: {:?}", &key.key() as &[u8]))
.map(|v| rlp::decode(&v).expect("decode db value failed") )
}
fn exists<T, R>(&self, col: Option<u32>, key: &Key<T, Target = R>) -> bool where R: Deref<Target = [u8]> {
let result = self.get(col, &key.key());
match result {
Ok(v) => v.is_some(),
Err(err) => {
panic!("db get failed, key: {:?}, err: {:?}", &key.key() as &[u8], err);
}
}
}
}

269
ethcore/db/src/keys.rs Normal file
View File

@@ -0,0 +1,269 @@
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Blockchain DB extras.
use std::io::Write;
use std::ops;
use common_types::BlockNumber;
use common_types::engines::epoch::Transition as EpochTransition;
use common_types::receipt::Receipt;
use ethereum_types::{H256, H264, U256};
use heapsize::HeapSizeOf;
use kvdb::PREFIX_LEN as DB_PREFIX_LEN;
use rlp;
use rlp_derive::{RlpEncodableWrapper, RlpDecodableWrapper, RlpEncodable, RlpDecodable};
use crate::db::Key;
/// Represents index of extra data in database
#[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)]
pub enum ExtrasIndex {
/// Block details index
BlockDetails = 0,
/// Block hash index
BlockHash = 1,
/// Transaction address index
TransactionAddress = 2,
/// Block receipts index
BlockReceipts = 4,
/// Epoch transition data index.
EpochTransitions = 5,
/// Pending epoch transition data index.
PendingEpochTransition = 6,
}
fn with_index(hash: &H256, i: ExtrasIndex) -> H264 {
let mut result = H264::default();
result[0] = i as u8;
(*result)[1..].clone_from_slice(hash);
result
}
/// Wrapper for block number used as a DB key.
pub struct BlockNumberKey([u8; 5]);
impl ops::Deref for BlockNumberKey {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Key<H256> for BlockNumber {
type Target = BlockNumberKey;
fn key(&self) -> Self::Target {
let mut result = [0u8; 5];
result[0] = ExtrasIndex::BlockHash as u8;
result[1] = (self >> 24) as u8;
result[2] = (self >> 16) as u8;
result[3] = (self >> 8) as u8;
result[4] = *self as u8;
BlockNumberKey(result)
}
}
impl Key<BlockDetails> for H256 {
type Target = H264;
fn key(&self) -> H264 {
with_index(self, ExtrasIndex::BlockDetails)
}
}
impl Key<TransactionAddress> for H256 {
type Target = H264;
fn key(&self) -> H264 {
with_index(self, ExtrasIndex::TransactionAddress)
}
}
impl Key<BlockReceipts> for H256 {
type Target = H264;
fn key(&self) -> H264 {
with_index(self, ExtrasIndex::BlockReceipts)
}
}
impl Key<common_types::engines::epoch::PendingTransition> for H256 {
type Target = H264;
fn key(&self) -> H264 {
with_index(self, ExtrasIndex::PendingEpochTransition)
}
}
/// length of epoch keys.
pub const EPOCH_KEY_LEN: usize = DB_PREFIX_LEN + 16;
/// epoch key prefix.
/// used to iterate over all epoch transitions in order from genesis.
pub const EPOCH_KEY_PREFIX: &'static [u8; DB_PREFIX_LEN] = &[
ExtrasIndex::EpochTransitions as u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
];
/// Epoch transitions key
pub struct EpochTransitionsKey([u8; EPOCH_KEY_LEN]);
impl ops::Deref for EpochTransitionsKey {
type Target = [u8];
fn deref(&self) -> &[u8] { &self.0[..] }
}
impl Key<EpochTransitions> for u64 {
type Target = EpochTransitionsKey;
fn key(&self) -> Self::Target {
let mut arr = [0u8; EPOCH_KEY_LEN];
arr[..DB_PREFIX_LEN].copy_from_slice(&EPOCH_KEY_PREFIX[..]);
write!(&mut arr[DB_PREFIX_LEN..], "{:016x}", self)
.expect("format arg is valid; no more than 16 chars will be written; qed");
EpochTransitionsKey(arr)
}
}
/// Familial details concerning a block
#[derive(Debug, Clone)]
pub struct BlockDetails {
/// Block number
pub number: BlockNumber,
/// Total difficulty of the block and all its parents
pub total_difficulty: U256,
/// Parent block hash
pub parent: H256,
/// List of children block hashes
pub children: Vec<H256>,
/// Whether the block is considered finalized
pub is_finalized: bool,
}
impl rlp::Encodable for BlockDetails {
fn rlp_append(&self, stream: &mut rlp::RlpStream) {
let use_short_version = !self.is_finalized;
match use_short_version {
true => { stream.begin_list(4); },
false => { stream.begin_list(5); },
}
stream.append(&self.number);
stream.append(&self.total_difficulty);
stream.append(&self.parent);
stream.append_list(&self.children);
if !use_short_version {
stream.append(&self.is_finalized);
}
}
}
impl rlp::Decodable for BlockDetails {
fn decode(rlp: &rlp::Rlp) -> Result<Self, rlp::DecoderError> {
let use_short_version = match rlp.item_count()? {
4 => true,
5 => false,
_ => return Err(rlp::DecoderError::RlpIncorrectListLen),
};
Ok(BlockDetails {
number: rlp.val_at(0)?,
total_difficulty: rlp.val_at(1)?,
parent: rlp.val_at(2)?,
children: rlp.list_at(3)?,
is_finalized: if use_short_version {
false
} else {
rlp.val_at(4)?
},
})
}
}
impl HeapSizeOf for BlockDetails {
fn heap_size_of_children(&self) -> usize {
self.children.heap_size_of_children()
}
}
/// Represents address of certain transaction within block
#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)]
pub struct TransactionAddress {
/// Block hash
pub block_hash: H256,
/// Transaction index within the block
pub index: usize
}
impl HeapSizeOf for TransactionAddress {
fn heap_size_of_children(&self) -> usize { 0 }
}
/// Contains all block receipts.
#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper)]
pub struct BlockReceipts {
/// Block receipts
pub receipts: Vec<Receipt>,
}
impl BlockReceipts {
/// Create new block receipts wrapper.
pub fn new(receipts: Vec<Receipt>) -> Self {
BlockReceipts {
receipts: receipts
}
}
}
impl HeapSizeOf for BlockReceipts {
fn heap_size_of_children(&self) -> usize {
self.receipts.heap_size_of_children()
}
}
/// Candidate transitions to an epoch with specific number.
#[derive(Clone, RlpEncodable, RlpDecodable)]
pub struct EpochTransitions {
/// Epoch number
pub number: u64,
/// List of candidate transitions
pub candidates: Vec<EpochTransition>,
}
#[cfg(test)]
mod tests {
use rlp::*;
use super::BlockReceipts;
#[test]
fn encode_block_receipts() {
let br = BlockReceipts::new(Vec::new());
let mut s = RlpStream::new_list(2);
s.append(&br);
assert!(!s.is_finished(), "List shouldn't finished yet");
s.append(&br);
assert!(s.is_finished(), "List should be finished now");
s.out();
}
}

26
ethcore/db/src/lib.rs Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Parity Ethereum database access utilities.
#![warn(missing_docs)]
mod db;
pub mod keys;
pub mod cache_manager;
pub use self::db::*;