Move ethcore files back into root.

This commit is contained in:
Gav Wood
2016-01-17 13:11:25 +01:00
parent 9b87bae322
commit 6ea8eaa3b5
159 changed files with 511 additions and 511 deletions

80
util/src/trie/journal.rs Normal file
View File

@@ -0,0 +1,80 @@
use sha3::*;
use hash::H256;
use bytes::*;
use rlp::*;
use hashdb::*;
/// Type of operation for the backing database - either a new node or a node deletion.
#[derive(Debug)]
enum Operation {
New(H256, Bytes),
Delete(H256),
}
/// How many insertions and removals were done in an `apply` operation.
pub struct Score {
/// Number of insertions.
pub inserts: usize,
/// Number of removals.
pub removes: usize,
}
/// A journal of operations on the backing database.
#[derive(Debug)]
pub struct Journal (Vec<Operation>);
impl Journal {
/// Create a new, empty, object.
pub fn new() -> Journal { Journal(vec![]) }
/// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal`
/// such that the reference is valid, once applied.
pub fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) {
if rlp.len() >= 32 {
let rlp_sha3 = rlp.sha3();
trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty());
out.append(&rlp_sha3);
self.0.push(Operation::New(rlp_sha3, rlp));
}
else {
trace!("new_node: inline node {:?}", rlp.pretty());
out.append_raw(&rlp, 1);
}
}
/// Given the RLP that encodes a now-unused node, leave `journal` in such a state that it is noted.
pub fn delete_node_sha3(&mut self, old_sha3: H256) {
trace!("delete_node: {:?}", old_sha3);
self.0.push(Operation::Delete(old_sha3));
}
/// Register an RLP-encoded node for deletion (given a slice), if it needs to be deleted.
pub fn delete_node(&mut self, old: &[u8]) {
let r = Rlp::new(old);
if r.is_data() && r.size() == 32 {
self.delete_node_sha3(r.as_val());
}
}
/// Apply this journal to the HashDB `db` and return the number of insertions and removals done.
pub fn apply(self, db: &mut HashDB) -> Score {
trace!("applying {:?} changes", self.0.len());
let mut ret = Score{inserts: 0, removes: 0};
for d in self.0.into_iter() {
match d {
Operation::Delete(h) => {
trace!("TrieDBMut::apply --- {:?}", &h);
db.remove(&h);
ret.removes += 1;
},
Operation::New(h, d) => {
trace!("TrieDBMut::apply +++ {:?} -> {:?}", &h, d.pretty());
db.emplace(h, d);
ret.inserts += 1;
}
}
}
ret
}
}

15
util/src/trie/mod.rs Normal file
View File

@@ -0,0 +1,15 @@
pub mod trietraits;
pub mod standardmap;
pub mod journal;
pub mod node;
pub mod triedb;
pub mod triedbmut;
pub mod sectriedb;
pub mod sectriedbmut;
pub use self::trietraits::*;
pub use self::standardmap::*;
pub use self::triedbmut::*;
pub use self::triedb::*;
pub use self::sectriedbmut::*;
pub use self::sectriedb::*;

121
util/src/trie/node.rs Normal file
View File

@@ -0,0 +1,121 @@
use hash::*;
use nibbleslice::*;
use bytes::*;
use rlp::*;
use super::journal::*;
/// Type of node in the trie and essential information thereof.
#[derive(Eq, PartialEq, Debug)]
pub enum Node<'a> {
Empty,
Leaf(NibbleSlice<'a>, &'a[u8]),
Extension(NibbleSlice<'a>, &'a[u8]),
Branch([&'a[u8]; 16], Option<&'a [u8]>)
}
impl<'a> Node<'a> {
/// Decode the `node_rlp` and return the Node.
pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> {
let r = Rlp::new(node_rlp);
match r.prototype() {
// either leaf or extension - decode first item with NibbleSlice::???
// and use is_leaf return to figure out which.
// if leaf, second item is a value (is_data())
// if extension, second item is a node (either SHA3 to be looked up and
// fed back into this function or inline RLP which can be fed back into this function).
Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
(slice, true) => Node::Leaf(slice, r.at(1).data()),
(slice, false) => Node::Extension(slice, r.at(1).as_raw()),
},
// branch - first 16 are nodes, 17th is a value (or empty).
Prototype::List(17) => {
let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() };
for i in 0..16 {
nodes[i] = r.at(i).as_raw();
}
Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
},
// an empty branch index.
Prototype::Data(0) => Node::Empty,
// something went wrong.
_ => panic!("Rlp is not valid.")
}
}
/// Encode the node into RLP.
///
/// Will always return the direct node RLP even if it's 32 or more bytes. To get the
/// RLP which would be valid for using in another node, use `encoded_and_added()`.
pub fn encoded(&self) -> Bytes {
match *self {
Node::Leaf(ref slice, ref value) => {
let mut stream = RlpStream::new_list(2);
stream.append(&slice.encoded(true));
stream.append(value);
stream.out()
},
Node::Extension(ref slice, ref raw_rlp) => {
let mut stream = RlpStream::new_list(2);
stream.append(&slice.encoded(false));
stream.append_raw(raw_rlp, 1);
stream.out()
},
Node::Branch(ref nodes, ref value) => {
let mut stream = RlpStream::new_list(17);
for i in 0..16 {
stream.append_raw(nodes[i], 1);
}
match *value {
Some(n) => { stream.append(&n); },
None => { stream.append_empty_data(); },
}
stream.out()
},
Node::Empty => {
let mut stream = RlpStream::new();
stream.append_empty_data();
stream.out()
}
}
}
/// Encode the node, adding it to `journal` if necessary and return the RLP valid for
/// insertion into a parent node.
pub fn encoded_and_added(&self, journal: &mut Journal) -> Bytes {
let mut stream = RlpStream::new();
match *self {
Node::Leaf(ref slice, ref value) => {
stream.append_list(2);
stream.append(&slice.encoded(true));
stream.append(value);
},
Node::Extension(ref slice, ref raw_rlp) => {
stream.append_list(2);
stream.append(&slice.encoded(false));
stream.append_raw(raw_rlp, 1);
},
Node::Branch(ref nodes, ref value) => {
stream.append_list(17);
for i in 0..16 {
stream.append_raw(nodes[i], 1);
}
match *value {
Some(n) => { stream.append(&n); },
None => { stream.append_empty_data(); },
}
},
Node::Empty => {
stream.append_empty_data();
}
}
let node = stream.out();
match node.len() {
0 ... 31 => node,
_ => {
let mut stream = RlpStream::new();
journal.new_node(node, &mut stream);
stream.out()
}
}
}
}

View File

@@ -0,0 +1,59 @@
use hash::*;
use sha3::*;
use hashdb::*;
use rlp::*;
use super::triedb::*;
use super::trietraits::*;
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
///
/// Use it as a `Trie` trait object. You can use `raw()` to get the backing TrieDB object.
pub struct SecTrieDB<'db> {
raw: TrieDB<'db>
}
impl<'db> SecTrieDB<'db> {
/// Create a new trie with the backing database `db` and empty `root`
/// Initialise to the state entailed by the genesis block.
/// This guarantees the trie is built correctly.
pub fn new(db: &'db HashDB, root: &'db H256) -> Self {
SecTrieDB { raw: TrieDB::new(db, root) }
}
/// Get a reference to the underlying raw TrieDB struct.
pub fn raw(&self) -> &TrieDB {
&self.raw
}
/// Get a mutable reference to the underlying raw TrieDB struct.
pub fn raw_mut(&mut self) -> &TrieDB {
&mut self.raw
}
}
impl<'db> Trie for SecTrieDB<'db> {
fn root(&self) -> &H256 { self.raw.root() }
fn contains(&self, key: &[u8]) -> bool {
self.raw.contains(&key.sha3())
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
self.raw.get(&key.sha3())
}
}
#[test]
fn trie_to_sectrie() {
use memorydb::*;
use super::triedbmut::*;
let mut memdb = MemoryDB::new();
let mut root = H256::new();
{
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]);
}
let t = SecTrieDB::new(&memdb, &root);
assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), &[0x01u8, 0x23]);
}

View File

@@ -0,0 +1,65 @@
use hash::*;
use sha3::*;
use hashdb::*;
use rlp::*;
use super::triedbmut::*;
use super::trietraits::*;
/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
///
/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing TrieDBMut object.
pub struct SecTrieDBMut<'db> {
raw: TrieDBMut<'db>
}
impl<'db> SecTrieDBMut<'db> {
/// Create a new trie with the backing database `db` and empty `root`
/// Initialise to the state entailed by the genesis block.
/// This guarantees the trie is built correctly.
pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
SecTrieDBMut { raw: TrieDBMut::new(db, root) }
}
/// Create a new trie with the backing database `db` and `root`
/// Panics, if `root` does not exist
pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self {
SecTrieDBMut { raw: TrieDBMut::from_existing(db, root) }
}
}
impl<'db> Trie for SecTrieDBMut<'db> {
fn root(&self) -> &H256 { self.raw.root() }
fn contains(&self, key: &[u8]) -> bool {
self.raw.contains(&key.sha3())
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
self.raw.get(&key.sha3())
}
}
impl<'db> TrieMut for SecTrieDBMut<'db> {
fn insert(&mut self, key: &[u8], value: &[u8]) {
self.raw.insert(&key.sha3(), value);
}
fn remove(&mut self, key: &[u8]) {
self.raw.remove(&key.sha3());
}
}
#[test]
fn sectrie_to_trie() {
use memorydb::*;
use super::triedb::*;
let mut memdb = MemoryDB::new();
let mut root = H256::new();
{
let mut t = SecTrieDBMut::new(&mut memdb, &mut root);
t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
}
let t = TrieDB::new(&memdb, &root);
assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]);
}

View File

@@ -0,0 +1,75 @@
//! Key-value datastore with a modified Merkle tree.
extern crate rand;
use bytes::*;
use sha3::*;
use hash::*;
/// Alphabet to use when creating words for insertion into tries.
pub enum Alphabet {
All,
Low,
Mid,
Custom(Bytes),
}
/// Standard test map for profiling tries.
pub struct StandardMap {
alphabet: Alphabet,
min_key: usize,
journal_key: usize,
count: usize,
}
impl StandardMap {
/// Get a bunch of random bytes, at least `min_count` bytes, at most `min_count` + `journal_count` bytes.
/// `seed` is mutated pseudoramdonly and used.
fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
seed.bytes()[0..r].to_vec()
}
/// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used.
fn random_value(seed: &mut H256) -> Bytes {
*seed = seed.sha3();
match seed.bytes()[0] % 2 {
1 => vec![seed.bytes()[31];1],
_ => seed.bytes().to_vec(),
}
}
/// Get a random word of, at least `min_count` bytes, at most `min_count` + `journal_count` bytes.
/// Each byte is an item from `alphabet`. `seed` is mutated pseudoramdonly and used.
fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec<u8> {
assert!(min_count + journal_count <= 32);
*seed = seed.sha3();
let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1));
let mut ret: Vec<u8> = Vec::with_capacity(r);
for i in 0..r {
ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]);
}
ret
}
/// Create the standard map (set of keys and values) for the object's fields.
pub fn make(&self) -> Vec<(Bytes, Bytes)> {
let low = b"abcdef";
let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_";
let mut d: Vec<(Bytes, Bytes)> = Vec::new();
let mut seed = H256::new();
for _ in 0..self.count {
let k = match self.alphabet {
Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed),
Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed),
Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed),
Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed),
};
let v = Self::random_value(&mut seed);
d.push((k, v))
}
d
}
}

220
util/src/trie/triedb.rs Normal file
View File

@@ -0,0 +1,220 @@
use common::*;
use hashdb::*;
use nibbleslice::*;
use rlp::*;
use super::trietraits::*;
use super::node::*;
/// A `Trie` implementation using a generic `HashDB` backing database.
///
/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys`
/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get
/// which items in the backing database do not belong to this trie. If this is the only trie in the
/// backing database, then `db_items_remaining()` should be empty.
///
/// # Example
/// ```
/// extern crate ethcore_util as util;
/// use util::trie::*;
/// use util::hashdb::*;
/// use util::memorydb::*;
/// use util::hash::*;
/// use util::rlp::*;
///
/// fn main() {
/// let mut memdb = MemoryDB::new();
/// let mut root = H256::new();
/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar");
/// let t = TrieDB::new(&memdb, &root);
/// assert!(t.contains(b"foo"));
/// assert_eq!(t.get(b"foo").unwrap(), b"bar");
/// assert!(t.db_items_remaining().is_empty());
/// }
/// ```
pub struct TrieDB<'db> {
db: &'db HashDB,
root: &'db H256,
pub hash_count: usize,
}
impl<'db> TrieDB<'db> {
/// Create a new trie with the backing database `db` and `root`
/// Panics, if `root` does not exist
pub fn new(db: &'db HashDB, root: &'db H256) -> Self {
if !db.exists(root) {
flush(format!("Trie root not found {}", root));
panic!("Trie root not found!");
}
TrieDB {
db: db,
root: root,
hash_count: 0
}
}
/// Get the backing database.
pub fn db(&'db self) -> &'db HashDB {
self.db
}
/// Determine all the keys in the backing database that belong to the trie.
pub fn keys(&self) -> Vec<H256> {
let mut ret: Vec<H256> = Vec::new();
ret.push(self.root.clone());
self.accumulate_keys(self.root_node(), &mut ret);
ret
}
/// Convert a vector of hashes to a hashmap of hash to occurances.
pub fn to_map(hashes: Vec<H256>) -> HashMap<H256, u32> {
let mut r: HashMap<H256, u32> = HashMap::new();
for h in hashes.into_iter() {
let c = *r.get(&h).unwrap_or(&0);
r.insert(h, c + 1);
}
r
}
/// Determine occurances of items in the backing database which are not related to this
/// trie.
pub fn db_items_remaining(&self) -> HashMap<H256, i32> {
let mut ret = self.db.keys();
for (k, v) in Self::to_map(self.keys()).into_iter() {
let keycount = *ret.get(&k).unwrap_or(&0);
match keycount <= v as i32 {
true => ret.remove(&k),
_ => ret.insert(k, keycount - v as i32),
};
}
ret
}
/// Recursion helper for `keys`.
fn accumulate_keys(&self, node: Node, acc: &mut Vec<H256>) {
let mut handle_payload = |payload| {
let p = Rlp::new(payload);
if p.is_data() && p.size() == 32 {
acc.push(p.as_val());
}
self.accumulate_keys(self.get_node(payload), acc);
};
match node {
Node::Extension(_, payload) => handle_payload(payload),
Node::Branch(payloads, _) => for payload in payloads.iter() { handle_payload(payload) },
_ => {},
}
}
/// Get the root node's RLP.
fn root_node(&self) -> Node {
Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!"))
}
/// Get the root node as a `Node`.
fn get_node<'a>(&'a self, node: &'a [u8]) -> Node {
Node::decoded(self.get_raw_or_lookup(node))
}
/// Indentation helper for `formal_all`.
fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
for _ in 0..size {
try!(write!(f, " "));
}
Ok(())
}
/// Recursion helper for implementation of formatting trait.
fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result {
match node {
Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())),
Node::Extension(ref slice, ref item) => {
try!(write!(f, "'{:?} ", slice));
try!(self.fmt_all(self.get_node(item), f, deepness));
},
Node::Branch(ref nodes, ref value) => {
try!(writeln!(f, ""));
match value {
&Some(v) => {
try!(self.fmt_indent(f, deepness + 1));
try!(writeln!(f, "=: {:?}", v.pretty()))
},
&None => {}
}
for i in 0..16 {
match self.get_node(nodes[i]) {
Node::Empty => {},
n => {
try!(self.fmt_indent(f, deepness + 1));
try!(write!(f, "'{:x} ", i));
try!(self.fmt_all(n, f, deepness + 1));
}
}
}
},
// empty
Node::Empty => {
try!(writeln!(f, "<empty>"));
}
};
Ok(())
}
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key {
let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!");
self.get_from_node(&root_rlp, key)
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
/// value exists for the key.
///
/// Note: Not a public API; use Trie trait functions.
fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key {
match Node::decoded(node) {
Node::Leaf(ref slice, ref value) if key == slice => Some(value),
Node::Extension(ref slice, ref item) if key.starts_with(slice) => {
self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len()))
},
Node::Branch(ref nodes, value) => match key.is_empty() {
true => value,
false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1))
},
_ => None
}
}
/// Given some node-describing data `node`, return the actual node RLP.
/// This could be a simple identity operation in the case that the node is sufficiently small, but
/// may require a database lookup.
fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] {
// check if its sha3 + len
let r = Rlp::new(node);
match r.is_data() && r.size() == 32 {
true => self.db.lookup(&r.as_val::<H256>()).expect("Not found!"),
false => node
}
}
}
impl<'db> Trie for TrieDB<'db> {
fn root(&self) -> &H256 { &self.root }
fn contains(&self, key: &[u8]) -> bool {
self.get(key).is_some()
}
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
self.do_lookup(&NibbleSlice::new(key))
}
}
impl<'db> fmt::Debug for TrieDB<'db> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
try!(writeln!(f, "c={:?} [", self.hash_count));
let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!");
try!(self.fmt_all(Node::decoded(root_rlp), f, 0));
writeln!(f, "]")
}
}

1100
util/src/trie/triedbmut.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,29 @@
use hash::H256;
use rlp::SHA3_NULL_RLP;
/// A key-value datastore implemented as a database-backed modified Merkle tree.
pub trait Trie {
/// Return the root of the trie.
fn root(&self) -> &H256;
/// Is the trie empty?
fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP }
/// Does the trie contain a given key?
fn contains(&self, key: &[u8]) -> bool;
/// What is the value of the given key in this trie?
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key;
}
/// A key-value datastore implemented as a database-backed modified Merkle tree.
pub trait TrieMut: Trie {
/// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing
/// `key` from the trie.
fn insert(&mut self, key: &[u8], value: &[u8]);
/// Remove a `key` from the trie. Equivalent to making it equal to the empty
/// value.
fn remove(&mut self, key: &[u8]);
}