grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into check-updates

Browse Source
This commit is contained in:
Gav Wood
2016-12-04 09:01:50 -08:00
parent 8e2aca719f 0c7b7fc8bc
commit 90a20a217f
280 changed files with 5818 additions and 4269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
util/src/common.rs
View File

@@ -33,6 +33,13 @@ macro_rules! vec_into {
}
}
#[macro_export]
macro_rules! slice_into {
( $( $x:expr ),* ) => {
&[ $( $x.into() ),* ]
}
}
#[macro_export]
macro_rules! hash_map {
() => { HashMap::new() };

15
util/src/hashdb.rs
View File

@@ -107,21 +107,6 @@ pub trait HashDB: AsHashDB + Send + Sync {
/// }
/// ```
fn remove(&mut self, key: &H256);
/// Insert auxiliary data into hashdb.
fn insert_aux(&mut self, _hash: Vec<u8>, _value: Vec<u8>) {
unimplemented!();
}
/// Get auxiliary data from hashdb.
fn get_aux(&self, _hash: &[u8]) -> Option<DBValue> {
unimplemented!();
}
/// Removes auxiliary data from hashdb.
fn remove_aux(&mut self, _hash: &[u8]) {
unimplemented!();
}
}
/// Upcast trait.

36
util/src/journaldb/archivedb.rs
View File

@@ -26,10 +26,6 @@ use kvdb::{Database, DBTransaction};
#[cfg(test)]
use std::env;
/// Suffix appended to auxiliary keys to distinguish them from normal keys.
/// Would be nich to use rocksdb columns for this eventually.
const AUX_FLAG: u8 = 255;
/// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
/// and latent-removal semantics.
///
@@ -108,26 +104,6 @@ impl HashDB for ArchiveDB {
fn remove(&mut self, key: &H256) {
self.overlay.remove(key);
}
fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
self.overlay.insert_aux(hash, value);
}
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
if let Some(res) = self.overlay.get_aux(hash) {
return Some(res)
}
let mut db_hash = hash.to_vec();
db_hash.push(AUX_FLAG);
self.backing.get(self.column, &db_hash)
.expect("Low-level database error. Some issue with your hard disk?")
}
fn remove_aux(&mut self, hash: &[u8]) {
self.overlay.remove_aux(hash);
}
}
impl JournalDB for ArchiveDB {
@@ -164,11 +140,6 @@ impl JournalDB for ArchiveDB {
}
}
for (mut key, value) in self.overlay.drain_aux() {
key.push(AUX_FLAG);
batch.put(self.column, &key, &value);
}
if self.latest_era.map_or(true, |e| now > e) {
batch.put(self.column, &LATEST_ERA_KEY, &encode(&now));
self.latest_era = Some(now);
@@ -204,11 +175,6 @@ impl JournalDB for ArchiveDB {
}
}
for (mut key, value) in self.overlay.drain_aux() {
key.push(AUX_FLAG);
batch.put(self.column, &key, &value);
}
Ok((inserts + deletes) as u32)
}
@@ -235,8 +201,8 @@ mod tests {
#![cfg_attr(feature="dev", allow(similar_names))]
use common::*;
use hashdb::{HashDB, DBValue};
use super::*;
use hashdb::*;
use journaldb::traits::JournalDB;
use kvdb::Database;

2
util/src/journaldb/earlymergedb.rs
View File

@@ -554,9 +554,9 @@ mod tests {
#![cfg_attr(feature="dev", allow(similar_names))]
use common::*;
use hashdb::{HashDB, DBValue};
use super::*;
use super::super::traits::JournalDB;
use hashdb::*;
use log::init_log;
use kvdb::{Database, DatabaseConfig};

2
util/src/journaldb/overlayrecentdb.rs
View File

@@ -422,7 +422,7 @@ mod tests {
use common::*;
use super::*;
use hashdb::*;
use hashdb::{HashDB, DBValue};
use log::init_log;
use journaldb::JournalDB;
use kvdb::Database;

2
util/src/journaldb/refcounteddb.rs
View File

@@ -215,9 +215,9 @@ mod tests {
#![cfg_attr(feature="dev", allow(similar_names))]
use common::*;
use hashdb::{HashDB, DBValue};
use super::*;
use super::super::traits::JournalDB;
use hashdb::*;
#[test]
fn long_history() {

2
util/src/kvdb.rs
View File

@@ -628,7 +628,7 @@ impl Drop for Database {
#[cfg(test)]
mod tests {
use hash::*;
use hash::H256;
use super::*;
use devtools::*;
use std::str::FromStr;

21
util/src/memorydb.rs
View File

@@ -17,7 +17,6 @@
//! Reference-counted memory-based `HashDB` implementation.
use hash::*;
use bytes::*;
use rlp::*;
use sha3::*;
use hashdb::*;
@@ -72,7 +71,6 @@ use std::collections::hash_map::Entry;
#[derive(Default, Clone, PartialEq)]
pub struct MemoryDB {
data: H256FastMap<(DBValue, i32)>,
aux: HashMap<Bytes, DBValue>,
}
impl MemoryDB {
@@ -80,7 +78,6 @@ impl MemoryDB {
pub fn new() -> MemoryDB {
MemoryDB {
data: H256FastMap::default(),
aux: HashMap::new(),
}
}
@@ -118,11 +115,6 @@ impl MemoryDB {
mem::replace(&mut self.data, H256FastMap::default())
}
/// Return the internal map of auxiliary data, clearing the current state.
pub fn drain_aux(&mut self) -> HashMap<Bytes, DBValue> {
mem::replace(&mut self.aux, HashMap::new())
}
/// Grab the raw information associated with a key. Returns None if the key
/// doesn't exist.
///
@@ -138,7 +130,6 @@ impl MemoryDB {
/// Returns the size of allocated heap memory
pub fn mem_used(&self) -> usize {
self.data.heap_size_of_children()
+ self.aux.heap_size_of_children()
}
/// Remove an element and delete it from storage if reference count reaches zero.
@@ -256,18 +247,6 @@ impl HashDB for MemoryDB {
self.data.insert(key.clone(), (DBValue::new(), -1));
}
}
fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
self.aux.insert(hash, DBValue::from_vec(value));
}
fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
self.aux.get(hash).cloned()
}
fn remove_aux(&mut self, hash: &[u8]) {
self.aux.remove(hash);
}
}
#[test]

10
util/src/stats.rs
View File

@@ -67,7 +67,7 @@ mod tests {
#[test]
fn check_histogram() {
let hist = Histogram::new(&vec_into![643,689,1408,2000,2296,2512,4250,4320,4842,4958,5804,6065,6098,6354,7002,7145,7845,8589,8593,8895], 5).unwrap();
let hist = Histogram::new(slice_into![643,689,1408,2000,2296,2512,4250,4320,4842,4958,5804,6065,6098,6354,7002,7145,7845,8589,8593,8895], 5).unwrap();
let correct_bounds: Vec<U256> = vec_into![643, 2294, 3945, 5596, 7247, 8898];
assert_eq!(Histogram { bucket_bounds: correct_bounds, counts: vec![4,2,4,6,4] }, hist);
}
@@ -75,7 +75,7 @@ mod tests {
#[test]
fn smaller_data_range_than_bucket_range() {
assert_eq!(
Histogram::new(&vec_into![1, 2, 2], 3),
Histogram::new(slice_into![1, 2, 2], 3),
Some(Histogram { bucket_bounds: vec_into![1, 2, 3, 4], counts: vec![1, 2, 0] })
);
}
@@ -83,7 +83,7 @@ mod tests {
#[test]
fn data_range_is_not_multiple_of_bucket_range() {
assert_eq!(
Histogram::new(&vec_into![1, 2, 5], 2),
Histogram::new(slice_into![1, 2, 5], 2),
Some(Histogram { bucket_bounds: vec_into![1, 4, 7], counts: vec![2, 1] })
);
}
@@ -91,13 +91,13 @@ mod tests {
#[test]
fn data_range_is_multiple_of_bucket_range() {
assert_eq!(
Histogram::new(&vec_into![1, 2, 6], 2),
Histogram::new(slice_into![1, 2, 6], 2),
Some(Histogram { bucket_bounds: vec_into![1, 4, 7], counts: vec![2, 1] })
);
}
#[test]
fn none_when_too_few_data() {
assert!(Histogram::new(&vec_into![], 1).is_none());
assert!(Histogram::new(slice_into![], 1).is_none());
}
}

13
util/src/trie/fatdb.rs
View File

@@ -17,7 +17,7 @@
use hash::H256;
use sha3::Hashable;
use hashdb::{HashDB, DBValue};
use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder};
use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder, TrieIterator};
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
/// Additionaly it stores inserted hash-key mappings for later retrieval.
@@ -46,7 +46,7 @@ impl<'db> FatDB<'db> {
}
impl<'db> Trie for FatDB<'db> {
fn iter<'a>(&'a self) -> super::Result<Box<Iterator<Item = TrieItem> + 'a>> {
fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
FatDBIterator::new(&self.raw).map(|iter| Box::new(iter) as Box<_>)
}
@@ -81,6 +81,12 @@ impl<'db> FatDBIterator<'db> {
}
}
impl<'db> TrieIterator for FatDBIterator<'db> {
fn seek(&mut self, key: &[u8]) -> super::Result<()> {
self.trie_iterator.seek(&key.sha3())
}
}
impl<'db> Iterator for FatDBIterator<'db> {
type Item = TrieItem<'db>;
@@ -88,7 +94,8 @@ impl<'db> Iterator for FatDBIterator<'db> {
self.trie_iterator.next()
.map(|res|
res.map(|(hash, value)| {
(self.trie.db().get_aux(&hash).expect("Missing fatdb hash").to_vec(), value)
let aux_hash = hash.sha3();
(self.trie.db().get(&aux_hash).expect("Missing fatdb hash").to_vec(), value)
})
)
}

10
util/src/trie/fatdbmut.rs
View File

@@ -51,6 +51,10 @@ impl<'db> FatDBMut<'db> {
pub fn db_mut(&mut self) -> &mut HashDB {
self.raw.db_mut()
}
fn to_aux_key(key: &[u8]) -> H256 {
key.sha3()
}
}
impl<'db> TrieMut for FatDBMut<'db> {
@@ -76,12 +80,14 @@ impl<'db> TrieMut for FatDBMut<'db> {
let hash = key.sha3();
try!(self.raw.insert(&hash, value));
let db = self.raw.db_mut();
db.insert_aux(hash.to_vec(), key.to_vec());
db.emplace(Self::to_aux_key(&hash), DBValue::from_slice(key));
Ok(())
}
fn remove(&mut self, key: &[u8]) -> super::Result<()> {
self.raw.remove(&key.sha3())
let hash = key.sha3();
self.raw.db_mut().remove(&Self::to_aux_key(&hash));
self.raw.remove(&hash)
}
}

10
util/src/trie/mod.rs
View File

@@ -102,7 +102,7 @@ pub trait Trie {
where 'a: 'b, R: Recorder;
/// Returns an iterator over elements of trie.
fn iter<'a>(&'a self) -> Result<Box<Iterator<Item = TrieItem> + 'a>>;
fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>>;
}
/// A key-value datastore implemented as a database-backed modified Merkle tree.
@@ -130,6 +130,12 @@ pub trait TrieMut {
fn remove(&mut self, key: &[u8]) -> Result<()>;
}
/// A trie iterator that also supports random access.
pub trait TrieIterator : Iterator {
/// Position the iterator on the first element with key > `key`
fn seek(&mut self, key: &[u8]) -> Result<()>;
}
/// Trie types
#[derive(Debug, PartialEq, Clone)]
pub enum TrieSpec {
@@ -193,7 +199,7 @@ impl<'db> Trie for TrieKinds<'db> {
wrapper!(self, get_recorded, key, r)
}
fn iter<'a>(&'a self) -> Result<Box<Iterator<Item = TrieItem> + 'a>> {
fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
wrapper!(self, iter,)
}
}

4
util/src/trie/sectriedb.rs
View File

@@ -18,7 +18,7 @@ use hash::H256;
use sha3::Hashable;
use hashdb::{HashDB, DBValue};
use super::triedb::TrieDB;
use super::{Trie, TrieItem, Recorder};
use super::{Trie, TrieItem, Recorder, TrieIterator};
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
///
@@ -49,7 +49,7 @@ impl<'db> SecTrieDB<'db> {
}
impl<'db> Trie for SecTrieDB<'db> {
fn iter<'a>(&'a self) -> super::Result<Box<Iterator<Item = TrieItem> + 'a>> {
fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
TrieDB::iter(&self.raw)
}

118
util/src/trie/triedb.rs
View File

@@ -20,7 +20,7 @@ use nibbleslice::*;
use rlp::*;
use super::node::Node;
use super::recorder::{Recorder, NoOp};
use super::{Trie, TrieItem, TrieError};
use super::{Trie, TrieItem, TrieError, TrieIterator};
/// A `Trie` implementation using a generic `HashDB` backing database.
///
@@ -295,6 +295,64 @@ impl<'a> TrieDBIterator<'a> {
Ok(r)
}
fn seek_descend<'key> ( &mut self, node: &[u8], key: &NibbleSlice<'key>, d: u32) -> super::Result<()> {
match Node::decoded(node) {
Node::Leaf(ref slice, _) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if slice == key {
self.trail.push(Crumb {
status: Status::At,
node: Node::decoded(node),
});
} else {
self.trail.push(Crumb {
status: Status::Exiting,
node: Node::decoded(node),
});
}
self.key_nibbles.extend(slice.iter());
Ok(())
},
Node::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) {
let mut r = NoOp;
self.trail.push(Crumb {
status: Status::At,
node: Node::decoded(node),
});
self.key_nibbles.extend(slice.iter());
let data = try!(self.db.get_raw_or_lookup(&*item, &mut r, d));
self.seek_descend(&data, &key.mid(slice.len()), d + 1)
} else {
try!(self.descend(node));
Ok(())
}
},
Node::Branch(ref nodes, _) => match key.is_empty() {
true => {
self.trail.push(Crumb {
status: Status::At,
node: Node::decoded(node),
});
Ok(())
},
false => {
let mut r = NoOp;
let i = key.at(0);
self.trail.push(Crumb {
status: Status::AtChild(i as usize),
node: Node::decoded(node),
});
self.key_nibbles.push(i);
let child = try!(self.db.get_raw_or_lookup(&*nodes[i as usize], &mut r, d));
self.seek_descend(&child, &key.mid(1), d + 1)
}
},
_ => Ok(())
}
}
/// Descend into a payload.
fn descend(&mut self, d: &[u8]) -> super::Result<()> {
self.trail.push(Crumb {
@@ -316,6 +374,17 @@ impl<'a> TrieDBIterator<'a> {
}
}
impl<'a> TrieIterator for TrieDBIterator<'a> {
/// Position the iterator on the first element with key >= `key`
fn seek(&mut self, key: &[u8]) -> super::Result<()> {
self.trail.clear();
self.key_nibbles.clear();
let mut r = NoOp;
let root_rlp = try!(self.db.root_data(&mut r));
self.seek_descend(&root_rlp, &NibbleSlice::new(key), 1)
}
}
impl<'a> Iterator for TrieDBIterator<'a> {
type Item = TrieItem<'a>;
@@ -372,7 +441,7 @@ impl<'a> Iterator for TrieDBIterator<'a> {
}
impl<'db> Trie for TrieDB<'db> {
fn iter<'a>(&'a self) -> super::Result<Box<Iterator<Item = TrieItem> + 'a>> {
fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>)
}
@@ -415,3 +484,48 @@ fn iterator() {
assert_eq!(d.iter().map(|i| i.clone().to_vec()).collect::<Vec<_>>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::<Vec<_>>());
assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::<Vec<_>>());
}
#[test]
fn iterator_seek() {
use memorydb::*;
use super::TrieMut;
use super::triedbmut::*;
let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ];
let mut memdb = MemoryDB::new();
let mut root = H256::new();
{
let mut t = TrieDBMut::new(&mut memdb, &mut root);
for x in &d {
t.insert(x, x).unwrap();
}
}
let t = TrieDB::new(&memdb, &root).unwrap();
let mut iter = t.iter().unwrap();
assert_eq!(iter.next(), Some(Ok((b"A".to_vec(), DBValue::from_slice(b"A")))));
iter.seek(b"!").unwrap();
assert_eq!(d, iter.map(|x| x.unwrap().1).collect::<Vec<_>>());
let mut iter = t.iter().unwrap();
iter.seek(b"A").unwrap();
assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"AA").unwrap();
assert_eq!(&d[2..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"A!").unwrap();
assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"AB").unwrap();
assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"AB!").unwrap();
assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"B").unwrap();
assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
let mut iter = t.iter().unwrap();
iter.seek(b"C").unwrap();
assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
}