Zero-alloc trie lookups (#3998)

* triedb cleanup

* factor out common portion of trie query

* allocate far fewer times in node decoding

* fix bench compilation

* introduce OwnedNode variant to make iter fast again

* generalize recorder trait to Query

* decode trie outputs cost-free in state

* test for passing closure as query
This commit is contained in:
Robert Habermeier 2017-01-06 16:18:45 +01:00 committed by Gav Wood
parent e983339edd
commit 9c00eb4e8a
13 changed files with 395 additions and 382 deletions

View File

@ -178,8 +178,8 @@ impl Account {
SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \ SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \
using it will not fail."); using it will not fail.");
let item: U256 = match db.get(key){ let item: U256 = match db.get_with(key, ::rlp::decode) {
Ok(x) => x.map_or_else(U256::zero, |v| decode(&*v)), Ok(x) => x.unwrap_or_else(U256::zero),
Err(e) => panic!("Encountered potential DB corruption: {}", e), Err(e) => panic!("Encountered potential DB corruption: {}", e),
}; };
let value: H256 = item.into(); let value: H256 = item.into();
@ -453,12 +453,12 @@ impl Account {
/// omitted. /// omitted.
pub fn prove_storage(&self, db: &HashDB, storage_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> { pub fn prove_storage(&self, db: &HashDB, storage_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> {
use util::trie::{Trie, TrieDB}; use util::trie::{Trie, TrieDB};
use util::trie::recorder::{Recorder, BasicRecorder as TrieRecorder}; use util::trie::recorder::Recorder;
let mut recorder = TrieRecorder::with_depth(from_level); let mut recorder = Recorder::with_depth(from_level);
let trie = TrieDB::new(db, &self.storage_root)?; let trie = TrieDB::new(db, &self.storage_root)?;
let _ = trie.get_recorded(&storage_key, &mut recorder)?; let _ = trie.get_with(&storage_key, &mut recorder)?;
Ok(recorder.drain().into_iter().map(|r| r.data).collect()) Ok(recorder.drain().into_iter().map(|r| r.data).collect())
} }

View File

@ -32,7 +32,7 @@ use state_db::StateDB;
use util::*; use util::*;
use util::trie::recorder::{Recorder, BasicRecorder as TrieRecorder}; use util::trie::recorder::Recorder;
mod account; mod account;
mod substate; mod substate;
@ -425,8 +425,8 @@ impl State {
// account is not found in the global cache, get from the DB and insert into local // account is not found in the global cache, get from the DB and insert into local
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
let maybe_acc = match db.get(address) { let maybe_acc = match db.get_with(address, Account::from_rlp) {
Ok(acc) => acc.map(|v| Account::from_rlp(&v)), Ok(acc) => acc,
Err(e) => panic!("Potential DB corruption encountered: {}", e), Err(e) => panic!("Potential DB corruption encountered: {}", e),
}; };
let r = maybe_acc.as_ref().map_or(H256::new(), |a| { let r = maybe_acc.as_ref().map_or(H256::new(), |a| {
@ -690,8 +690,8 @@ impl State {
// not found in the global cache, get from the DB and insert into local // not found in the global cache, get from the DB and insert into local
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
let mut maybe_acc = match db.get(a) { let mut maybe_acc = match db.get_with(a, Account::from_rlp) {
Ok(acc) => acc.map(|v| Account::from_rlp(&v)), Ok(acc) => acc,
Err(e) => panic!("Potential DB corruption encountered: {}", e), Err(e) => panic!("Potential DB corruption encountered: {}", e),
}; };
if let Some(ref mut account) = maybe_acc.as_mut() { if let Some(ref mut account) = maybe_acc.as_mut() {
@ -722,9 +722,8 @@ impl State {
None => { None => {
let maybe_acc = if self.db.check_non_null_bloom(a) { let maybe_acc = if self.db.check_non_null_bloom(a) {
let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
match db.get(a) { match db.get_with(a, Account::from_rlp) {
Ok(Some(acc)) => AccountEntry::new_clean(Some(Account::from_rlp(&acc))), Ok(acc) => AccountEntry::new_clean(acc),
Ok(None) => AccountEntry::new_clean(None),
Err(e) => panic!("Potential DB corruption encountered: {}", e), Err(e) => panic!("Potential DB corruption encountered: {}", e),
} }
} else { } else {
@ -770,9 +769,9 @@ impl State {
/// Requires a secure trie to be used for accurate results. /// Requires a secure trie to be used for accurate results.
/// `account_key` == sha3(address) /// `account_key` == sha3(address)
pub fn prove_account(&self, account_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> { pub fn prove_account(&self, account_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> {
let mut recorder = TrieRecorder::with_depth(from_level); let mut recorder = Recorder::with_depth(from_level);
let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
let _ = trie.get_recorded(&account_key, &mut recorder)?; trie.get_with(&account_key, &mut recorder)?;
Ok(recorder.drain().into_iter().map(|r| r.data).collect()) Ok(recorder.drain().into_iter().map(|r| r.data).collect())
} }
@ -786,8 +785,8 @@ impl State {
// TODO: probably could look into cache somehow but it's keyed by // TODO: probably could look into cache somehow but it's keyed by
// address, not sha3(address). // address, not sha3(address).
let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
let acc = match trie.get(&account_key)? { let acc = match trie.get_with(&account_key, Account::from_rlp)? {
Some(rlp) => Account::from_rlp(&rlp), Some(acc) => acc,
None => return Ok(Vec::new()), None => return Ok(Vec::new()),
}; };
@ -799,8 +798,8 @@ impl State {
/// Only works when backed by a secure trie. /// Only works when backed by a secure trie.
pub fn code_by_address_hash(&self, account_key: H256) -> Result<Option<Bytes>, Box<TrieError>> { pub fn code_by_address_hash(&self, account_key: H256) -> Result<Option<Bytes>, Box<TrieError>> {
let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
let mut acc = match trie.get(&account_key)? { let mut acc = match trie.get_with(&account_key, Account::from_rlp)? {
Some(rlp) => Account::from_rlp(&rlp), Some(acc) => acc,
None => return Ok(None), None => return Ok(None),
}; };

View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>. // along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! benchmarking for rlp //! benchmarking for bigint
//! should be started with: //! should be started with:
//! ```bash //! ```bash
//! multirust run nightly cargo bench //! multirust run nightly cargo bench
@ -24,10 +24,10 @@
#![feature(asm)] #![feature(asm)]
extern crate test; extern crate test;
extern crate ethcore_bigint as bigint; extern crate ethcore_util;
use test::{Bencher, black_box}; use test::{Bencher, black_box};
use bigint::uint::{U256, U512, Uint, U128}; use ethcore_util::{U256, U512, Uint, U128};
#[bench] #[bench]
fn u256_add(b: &mut Bencher) { fn u256_add(b: &mut Bencher) {

View File

@ -24,12 +24,12 @@
extern crate test; extern crate test;
extern crate rlp; extern crate rlp;
extern crate ethcore_bigint as bigint; extern crate ethcore_util as util;
use test::Bencher; use test::Bencher;
use std::str::FromStr; use std::str::FromStr;
use rlp::*; use rlp::*;
use bigint::uint::U256; use util::U256;
#[bench] #[bench]
fn bench_stream_u64_value(b: &mut Bencher) { fn bench_stream_u64_value(b: &mut Bencher) {

View File

@ -18,27 +18,26 @@
//! An owning, nibble-oriented byte vector. //! An owning, nibble-oriented byte vector.
use ::NibbleSlice; use ::NibbleSlice;
use elastic_array::ElasticArray36;
#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Debug)]
/// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`. /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`.
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct NibbleVec { pub struct NibbleVec {
inner: Vec<u8>, inner: ElasticArray36<u8>,
len: usize, len: usize,
} }
impl Default for NibbleVec {
fn default() -> Self {
NibbleVec::new()
}
}
impl NibbleVec { impl NibbleVec {
/// Make a new `NibbleVec` /// Make a new `NibbleVec`
pub fn new() -> Self { pub fn new() -> Self {
NibbleVec { NibbleVec {
inner: Vec::new(), inner: ElasticArray36::new(),
len: 0
}
}
/// Make a `NibbleVec` with capacity for `n` nibbles.
pub fn with_capacity(n: usize) -> Self {
NibbleVec {
inner: Vec::with_capacity((n / 2) + (n % 2)),
len: 0 len: 0
} }
} }
@ -49,9 +48,6 @@ impl NibbleVec {
/// Retrurns true if `NibbleVec` has zero length /// Retrurns true if `NibbleVec` has zero length
pub fn is_empty(&self) -> bool { self.len == 0 } pub fn is_empty(&self) -> bool { self.len == 0 }
/// Capacity of the `NibbleVec`.
pub fn capacity(&self) -> usize { self.inner.capacity() * 2 }
/// Try to get the nibble at the given offset. /// Try to get the nibble at the given offset.
pub fn at(&self, idx: usize) -> u8 { pub fn at(&self, idx: usize) -> u8 {
if idx % 2 == 0 { if idx % 2 == 0 {
@ -109,7 +105,7 @@ impl NibbleVec {
impl<'a> From<NibbleSlice<'a>> for NibbleVec { impl<'a> From<NibbleSlice<'a>> for NibbleVec {
fn from(s: NibbleSlice<'a>) -> Self { fn from(s: NibbleSlice<'a>) -> Self {
let mut v = NibbleVec::with_capacity(s.len()); let mut v = NibbleVec::new();
for i in 0..s.len() { for i in 0..s.len() {
v.push(s.at(i)); v.push(s.at(i));
} }

View File

@ -16,8 +16,8 @@
use hash::H256; use hash::H256;
use sha3::Hashable; use sha3::Hashable;
use hashdb::{HashDB, DBValue}; use hashdb::HashDB;
use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder, TrieIterator}; use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query};
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
/// Additionaly it stores inserted hash-key mappings for later retrieval. /// Additionaly it stores inserted hash-key mappings for later retrieval.
@ -58,10 +58,10 @@ impl<'db> Trie for FatDB<'db> {
self.raw.contains(&key.sha3()) self.raw.contains(&key.sha3())
} }
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>> fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
where 'a: 'b, R: Recorder where 'a: 'key
{ {
self.raw.get_recorded(&key.sha3(), rec) self.raw.get_with(&key.sha3(), query)
} }
} }
@ -104,6 +104,7 @@ impl<'db> Iterator for FatDBIterator<'db> {
#[test] #[test]
fn fatdb_to_trie() { fn fatdb_to_trie() {
use memorydb::MemoryDB; use memorydb::MemoryDB;
use hashdb::DBValue;
use trie::{FatDBMut, TrieMut}; use trie::{FatDBMut, TrieMut};
let mut memdb = MemoryDB::new(); let mut memdb = MemoryDB::new();

94
util/src/trie/lookup.rs Normal file
View File

@ -0,0 +1,94 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Trie lookup via HashDB.
use hashdb::HashDB;
use nibbleslice::NibbleSlice;
use rlp::{Rlp, View};
use ::{H256};
use super::{TrieError, Query};
use super::node::Node;
/// Trie lookup helper object.
pub struct Lookup<'a, Q: Query> {
/// database to query from.
pub db: &'a HashDB,
/// Query object to record nodes and transform data.
pub query: Q,
/// Hash to start at
pub hash: H256,
}
impl<'a, Q: Query> Lookup<'a, Q> {
/// Look up the given key. If the value is found, it will be passed to the given
/// function to decode or copy.
pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result<Option<Q::Item>> {
let mut hash = self.hash;
// this loop iterates through non-inline nodes.
for depth in 0.. {
let node_data = match self.db.get(&hash) {
Some(value) => value,
None => return Err(Box::new(match depth {
0 => TrieError::InvalidStateRoot(hash),
_ => TrieError::IncompleteDatabase(hash),
})),
};
self.query.record(&hash, &node_data, depth);
// this loop iterates through all inline children (usually max 1)
// without incrementing the depth.
let mut node_data = &node_data[..];
loop {
match Node::decoded(node_data) {
Node::Leaf(slice, value) => {
return Ok(match slice == key {
true => Some(self.query.decode(value)),
false => None,
})
}
Node::Extension(slice, item) => {
if key.starts_with(&slice) {
node_data = item;
key = key.mid(slice.len());
} else {
return Ok(None)
}
}
Node::Branch(children, value) => match key.is_empty() {
true => return Ok(value.map(move |val| self.query.decode(val))),
false => {
node_data = children[key.at(0) as usize];
key = key.mid(1);
}
},
_ => return Ok(None),
}
// check if new node data is inline or hash.
let r = Rlp::new(node_data);
if r.is_data() && r.size() == 32 {
hash = r.as_val();
break
}
}
}
Ok(None)
}
}

View File

@ -38,6 +38,7 @@ pub mod recorder;
mod fatdb; mod fatdb;
mod fatdbmut; mod fatdbmut;
mod lookup;
pub use self::standardmap::{Alphabet, StandardMap, ValueMode}; pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
pub use self::triedbmut::TrieDBMut; pub use self::triedbmut::TrieDBMut;
@ -76,6 +77,46 @@ pub type Result<T> = ::std::result::Result<T, Box<TrieError>>;
/// Trie-Item type. /// Trie-Item type.
pub type TrieItem<'a> = Result<(Vec<u8>, DBValue)>; pub type TrieItem<'a> = Result<(Vec<u8>, DBValue)>;
/// Description of what kind of query will be made to the trie.
///
/// This is implemented for any &mut recorder (where the query will return
/// a DBValue), any function taking raw bytes (where no recording will be made),
/// or any tuple of (&mut Recorder, FnOnce(&[u8]))
pub trait Query {
/// Output item.
type Item;
/// Decode a byte-slice into the desired item.
fn decode(self, &[u8]) -> Self::Item;
/// Record that a node has been passed through.
fn record(&mut self, &H256, &[u8], u32) { }
}
impl<'a> Query for &'a mut Recorder {
type Item = DBValue;
fn decode(self, value: &[u8]) -> DBValue { DBValue::from_slice(value) }
fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
(&mut **self).record(hash, data, depth);
}
}
impl<F, T> Query for F where F: for<'a> FnOnce(&'a [u8]) -> T {
type Item = T;
fn decode(self, value: &[u8]) -> T { (self)(value) }
}
impl<'a, F, T> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T {
type Item = T;
fn decode(self, value: &[u8]) -> T { (self.1)(value) }
fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
self.0.record(hash, data, depth)
}
}
/// A key-value datastore implemented as a database-backed modified Merkle tree. /// A key-value datastore implemented as a database-backed modified Merkle tree.
pub trait Trie { pub trait Trie {
/// Return the root of the trie. /// Return the root of the trie.
@ -91,13 +132,13 @@ pub trait Trie {
/// What is the value of the given key in this trie? /// What is the value of the given key in this trie?
fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key {
self.get_recorded(key, &mut recorder::NoOp) self.get_with(key, DBValue::from_slice)
} }
/// Query the value of the given key in this trie while recording visited nodes /// Search for the key with the given query parameter. See the docs of the `Query`
/// to the given recorder. If the query encounters an error, the nodes passed to the recorder are unspecified. /// trait for more details.
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<DBValue>> fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q)
where 'a: 'b, R: Recorder; -> Result<Option<Q::Item>> where 'a: 'key;
/// Returns a depth-first iterator over the elements of trie. /// Returns a depth-first iterator over the elements of trie.
fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>>; fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>>;
@ -192,9 +233,10 @@ impl<'db> Trie for TrieKinds<'db> {
wrapper!(self, contains, key) wrapper!(self, contains, key)
} }
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<DBValue>> fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result<Option<Q::Item>>
where 'a: 'b, R: Recorder { where 'a: 'key
wrapper!(self, get_recorded, key, r) {
wrapper!(self, get_with, key, query)
} }
fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>> { fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>> {

View File

@ -16,6 +16,7 @@
use elastic_array::ElasticArray36; use elastic_array::ElasticArray36;
use nibbleslice::*; use nibbleslice::*;
use nibblevec::NibbleVec;
use bytes::*; use bytes::*;
use rlp::*; use rlp::*;
use hashdb::DBValue; use hashdb::DBValue;
@ -24,40 +25,21 @@ use hashdb::DBValue;
pub type NodeKey = ElasticArray36<u8>; pub type NodeKey = ElasticArray36<u8>;
/// Type of node in the trie and essential information thereof. /// Type of node in the trie and essential information thereof.
#[derive(Eq, PartialEq, Debug)] #[derive(Eq, PartialEq, Debug, Clone)]
pub enum Node { pub enum Node<'a> {
/// Null trie node; could be an empty root or an empty branch entry. /// Null trie node; could be an empty root or an empty branch entry.
Empty, Empty,
/// Leaf node; has key slice and value. Value may not be empty. /// Leaf node; has key slice and value. Value may not be empty.
Leaf(NodeKey, DBValue), Leaf(NibbleSlice<'a>, &'a [u8]),
/// Extension node; has key slice and node data. Data may not be null. /// Extension node; has key slice and node data. Data may not be null.
Extension(NodeKey, DBValue), Extension(NibbleSlice<'a>, &'a [u8]),
/// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data. /// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data.
Branch([NodeKey; 16], Option<DBValue>) Branch([&'a [u8]; 16], Option<&'a [u8]>)
} }
impl Clone for Node { impl<'a> Node<'a> {
fn clone(&self) -> Node {
match *self {
Node::Empty => Node::Empty,
Node::Leaf(ref k, ref v) => Node::Leaf(k.clone(), v.clone()),
Node::Extension(ref k, ref v) => Node::Extension(k.clone(), v.clone()),
Node::Branch(ref k, ref v) => {
let mut branch = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
for i in 0 .. 16 {
branch[i] = k[i].clone();
}
Node::Branch(branch, v.clone())
}
}
}
}
impl Node {
/// Decode the `node_rlp` and return the Node. /// Decode the `node_rlp` and return the Node.
pub fn decoded(node_rlp: &[u8]) -> Node { pub fn decoded(node_rlp: &'a [u8]) -> Self {
let r = Rlp::new(node_rlp); let r = Rlp::new(node_rlp);
match r.prototype() { match r.prototype() {
// either leaf or extension - decode first item with NibbleSlice::??? // either leaf or extension - decode first item with NibbleSlice::???
@ -66,18 +48,16 @@ impl Node {
// if extension, second item is a node (either SHA3 to be looked up and // if extension, second item is a node (either SHA3 to be looked up and
// fed back into this function or inline RLP which can be fed back into this function). // fed back into this function or inline RLP which can be fed back into this function).
Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
(slice, true) => Node::Leaf(slice.encoded(true), DBValue::from_slice(r.at(1).data())), (slice, true) => Node::Leaf(slice, r.at(1).data()),
(slice, false) => Node::Extension(slice.encoded(false), DBValue::from_slice(r.at(1).as_raw())), (slice, false) => Node::Extension(slice, r.at(1).as_raw()),
}, },
// branch - first 16 are nodes, 17th is a value (or empty). // branch - first 16 are nodes, 17th is a value (or empty).
Prototype::List(17) => { Prototype::List(17) => {
let mut nodes: [NodeKey; 16] = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), let mut nodes = [&[] as &[u8]; 16];
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
for i in 0..16 { for i in 0..16 {
nodes[i] = NodeKey::from_slice(r.at(i).as_raw()); nodes[i] = r.at(i).as_raw();
} }
Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(DBValue::from_slice(r.at(16).data())) }) Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
}, },
// an empty branch index. // an empty branch index.
Prototype::Data(0) => Node::Empty, Prototype::Data(0) => Node::Empty,
@ -94,23 +74,23 @@ impl Node {
match *self { match *self {
Node::Leaf(ref slice, ref value) => { Node::Leaf(ref slice, ref value) => {
let mut stream = RlpStream::new_list(2); let mut stream = RlpStream::new_list(2);
stream.append(&&**slice); stream.append(&&*slice.encoded(true));
stream.append(&&**value); stream.append(value);
stream.out() stream.out()
}, },
Node::Extension(ref slice, ref raw_rlp) => { Node::Extension(ref slice, ref raw_rlp) => {
let mut stream = RlpStream::new_list(2); let mut stream = RlpStream::new_list(2);
stream.append(&&**slice); stream.append(&&*slice.encoded(false));
stream.append_raw(&&*raw_rlp, 1); stream.append_raw(raw_rlp, 1);
stream.out() stream.out()
}, },
Node::Branch(ref nodes, ref value) => { Node::Branch(ref nodes, ref value) => {
let mut stream = RlpStream::new_list(17); let mut stream = RlpStream::new_list(17);
for i in 0..16 { for i in 0..16 {
stream.append_raw(&*nodes[i], 1); stream.append_raw(nodes[i], 1);
} }
match *value { match *value {
Some(ref n) => { stream.append(&&**n); }, Some(ref n) => { stream.append(n); },
None => { stream.append_empty_data(); }, None => { stream.append_empty_data(); },
} }
stream.out() stream.out()
@ -123,3 +103,64 @@ impl Node {
} }
} }
} }
/// An owning node type. Useful for trie iterators.
#[derive(Debug, PartialEq, Eq)]
pub enum OwnedNode {
/// Empty trie node.
Empty,
/// Leaf node: partial key and value.
Leaf(NibbleVec, DBValue),
/// Extension node: partial key and child node.
Extension(NibbleVec, DBValue),
/// Branch node: 16 children and an optional value.
Branch([NodeKey; 16], Option<DBValue>),
}
impl Clone for OwnedNode {
fn clone(&self) -> Self {
match *self {
OwnedNode::Empty => OwnedNode::Empty,
OwnedNode::Leaf(ref k, ref v) => OwnedNode::Leaf(k.clone(), v.clone()),
OwnedNode::Extension(ref k, ref c) => OwnedNode::Extension(k.clone(), c.clone()),
OwnedNode::Branch(ref c, ref v) => {
let mut children = [
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
];
for (owned, borrowed) in children.iter_mut().zip(c.iter()) {
*owned = borrowed.clone()
}
OwnedNode::Branch(children, v.as_ref().cloned())
}
}
}
}
impl<'a> From<Node<'a>> for OwnedNode {
fn from(node: Node<'a>) -> Self {
match node {
Node::Empty => OwnedNode::Empty,
Node::Leaf(k, v) => OwnedNode::Leaf(k.into(), DBValue::from_slice(v)),
Node::Extension(k, child) => OwnedNode::Extension(k.into(), DBValue::from_slice(child)),
Node::Branch(c, val) => {
let mut children = [
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
];
for (owned, borrowed) in children.iter_mut().zip(c.iter()) {
*owned = NodeKey::from_slice(borrowed)
}
OwnedNode::Branch(children, val.map(DBValue::from_slice))
}
}
}
}

View File

@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>. // along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Trie query recorder.
use sha3::Hashable; use sha3::Hashable;
use {Bytes, H256}; use {Bytes, H256};
@ -30,63 +32,36 @@ pub struct Record {
pub hash: H256, pub hash: H256,
} }
/// Trie node recorder. /// Records trie nodes as they pass it.
///
/// These are used to record which nodes are visited during a trie query.
/// Inline nodes are not to be recorded, as they are contained within their parent.
pub trait Recorder {
/// Record that the given node has been visited.
///
/// The depth parameter is the depth of the visited node, with the root node having depth 0.
fn record(&mut self, hash: &H256, data: &[u8], depth: u32);
/// Drain all accepted records from the recorder in ascending order by depth.
fn drain(&mut self) -> Vec<Record> where Self: Sized;
}
/// A no-op trie recorder. This ignores everything which is thrown at it.
pub struct NoOp;
impl Recorder for NoOp {
#[inline]
fn record(&mut self, _hash: &H256, _data: &[u8], _depth: u32) {}
#[inline]
fn drain(&mut self) -> Vec<Record> { Vec::new() }
}
/// A simple recorder. Does nothing fancy but fulfills the `Recorder` interface
/// properly.
#[derive(Debug)] #[derive(Debug)]
pub struct BasicRecorder { pub struct Recorder {
nodes: Vec<Record>, nodes: Vec<Record>,
min_depth: u32, min_depth: u32,
} }
impl Default for BasicRecorder { impl Default for Recorder {
fn default() -> Self { fn default() -> Self {
BasicRecorder::new() Recorder::new()
} }
} }
impl BasicRecorder { impl Recorder {
/// Create a new `BasicRecorder` which records all given nodes. /// Create a new `Recorder` which records all given nodes.
#[inline] #[inline]
pub fn new() -> Self { pub fn new() -> Self {
BasicRecorder::with_depth(0) Recorder::with_depth(0)
} }
/// Create a `BasicRecorder` which only records nodes beyond a given depth. /// Create a `Recorder` which only records nodes beyond a given depth.
pub fn with_depth(depth: u32) -> Self { pub fn with_depth(depth: u32) -> Self {
BasicRecorder { Recorder {
nodes: Vec::new(), nodes: Vec::new(),
min_depth: depth, min_depth: depth,
} }
} }
}
impl Recorder for BasicRecorder { /// Record a visited node, given its hash, data, and depth.
fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
debug_assert_eq!(data.sha3(), *hash); debug_assert_eq!(data.sha3(), *hash);
if depth >= self.min_depth { if depth >= self.min_depth {
@ -98,7 +73,8 @@ impl Recorder for BasicRecorder {
} }
} }
fn drain(&mut self) -> Vec<Record> { /// Drain all visited records.
pub fn drain(&mut self) -> Vec<Record> {
::std::mem::replace(&mut self.nodes, Vec::new()) ::std::mem::replace(&mut self.nodes, Vec::new())
} }
} }
@ -109,20 +85,9 @@ mod tests {
use sha3::Hashable; use sha3::Hashable;
use ::H256; use ::H256;
#[test]
fn no_op_does_nothing() {
let mut no_op = NoOp;
let (node1, node2) = (&[1], &[2]);
let (hash1, hash2) = (node1.sha3(), node2.sha3());
no_op.record(&hash1, node1, 1);
no_op.record(&hash2, node2, 2);
assert_eq!(no_op.drain(), Vec::new());
}
#[test] #[test]
fn basic_recorder() { fn basic_recorder() {
let mut basic = BasicRecorder::new(); let mut basic = Recorder::new();
let node1 = vec![1, 2, 3, 4]; let node1 = vec![1, 2, 3, 4];
let node2 = vec![4, 5, 6, 7, 8, 9, 10]; let node2 = vec![4, 5, 6, 7, 8, 9, 10];
@ -148,7 +113,7 @@ mod tests {
#[test] #[test]
fn basic_recorder_min_depth() { fn basic_recorder_min_depth() {
let mut basic = BasicRecorder::with_depth(400); let mut basic = Recorder::with_depth(400);
let node1 = vec![1, 2, 3, 4]; let node1 = vec![1, 2, 3, 4];
let node2 = vec![4, 5, 6, 7, 8, 9, 10]; let node2 = vec![4, 5, 6, 7, 8, 9, 10];
@ -192,9 +157,9 @@ mod tests {
} }
let trie = TrieDB::new(&db, &root).unwrap(); let trie = TrieDB::new(&db, &root).unwrap();
let mut recorder = BasicRecorder::new(); let mut recorder = Recorder::new();
trie.get_recorded(b"pirate", &mut recorder).unwrap().unwrap(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap();
let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect();
assert_eq!(nodes, vec![ assert_eq!(nodes, vec![
@ -213,7 +178,7 @@ mod tests {
] ]
]); ]);
trie.get_recorded(b"letter", &mut recorder).unwrap().unwrap(); trie.get_with(b"letter", &mut recorder).unwrap().unwrap();
let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect();
assert_eq!(nodes, vec![ assert_eq!(nodes, vec![

View File

@ -16,9 +16,9 @@
use hash::H256; use hash::H256;
use sha3::Hashable; use sha3::Hashable;
use hashdb::{HashDB, DBValue}; use hashdb::HashDB;
use super::triedb::TrieDB; use super::triedb::TrieDB;
use super::{Trie, TrieItem, Recorder, TrieIterator}; use super::{Trie, TrieItem, TrieIterator, Query};
/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
/// ///
@ -59,16 +59,17 @@ impl<'db> Trie for SecTrieDB<'db> {
self.raw.contains(&key.sha3()) self.raw.contains(&key.sha3())
} }
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>> fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
where 'a: 'b, R: Recorder where 'a: 'key
{ {
self.raw.get_recorded(&key.sha3(), rec) self.raw.get_with(&key.sha3(), query)
} }
} }
#[test] #[test]
fn trie_to_sectrie() { fn trie_to_sectrie() {
use memorydb::MemoryDB; use memorydb::MemoryDB;
use hashdb::DBValue;
use super::triedbmut::TrieDBMut; use super::triedbmut::TrieDBMut;
use super::super::TrieMut; use super::super::TrieMut;

View File

@ -18,16 +18,14 @@ use common::*;
use hashdb::*; use hashdb::*;
use nibbleslice::*; use nibbleslice::*;
use rlp::*; use rlp::*;
use super::node::Node; use super::node::{Node, OwnedNode};
use super::recorder::{Recorder, NoOp}; use super::lookup::Lookup;
use super::{Trie, TrieItem, TrieError, TrieIterator}; use super::{Trie, TrieItem, TrieError, TrieIterator, Query};
/// A `Trie` implementation using a generic `HashDB` backing database. /// A `Trie` implementation using a generic `HashDB` backing database.
/// ///
/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` /// Use it as a `Trie` trait object. You can use `db()` to get the backing database object.
/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get /// Use `get` and `contains` to query values associated with keys in the trie.
/// which items in the backing database do not belong to this trie. If this is the only trie in the
/// backing database, then `db_items_remaining()` should be empty.
/// ///
/// # Example /// # Example
/// ``` /// ```
@ -45,7 +43,6 @@ use super::{Trie, TrieItem, TrieError, TrieIterator};
/// let t = TrieDB::new(&memdb, &root).unwrap(); /// let t = TrieDB::new(&memdb, &root).unwrap();
/// assert!(t.contains(b"foo").unwrap()); /// assert!(t.contains(b"foo").unwrap());
/// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar")); /// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
/// assert!(t.db_items_remaining().unwrap().is_empty());
/// } /// }
/// ``` /// ```
pub struct TrieDB<'db> { pub struct TrieDB<'db> {
@ -76,74 +73,12 @@ impl<'db> TrieDB<'db> {
self.db self.db
} }
/// Determine all the keys in the backing database that belong to the trie.
pub fn keys(&self) -> super::Result<Vec<H256>> {
let mut ret: Vec<H256> = Vec::new();
ret.push(self.root.clone());
self.accumulate_keys(self.root_node(&mut NoOp)?, &mut ret)?;
Ok(ret)
}
/// Convert a vector of hashes to a hashmap of hash to occurrences.
pub fn to_map(hashes: Vec<H256>) -> HashMap<H256, u32> {
let mut r: HashMap<H256, u32> = HashMap::new();
for h in hashes {
*r.entry(h).or_insert(0) += 1;
}
r
}
/// Determine occurrences of items in the backing database which are not related to this
/// trie.
pub fn db_items_remaining(&self) -> super::Result<HashMap<H256, i32>> {
let mut ret = self.db.keys();
for (k, v) in Self::to_map(self.keys()?) {
let keycount = *ret.get(&k).unwrap_or(&0);
match keycount <= v as i32 {
true => ret.remove(&k),
_ => ret.insert(k, keycount - v as i32),
};
}
Ok(ret)
}
/// Recursion helper for `keys`.
fn accumulate_keys(&self, node: Node, acc: &mut Vec<H256>) -> super::Result<()> {
let mut handle_payload = |payload| {
let p = Rlp::new(payload);
if p.is_data() && p.size() == 32 {
acc.push(p.as_val());
}
self.accumulate_keys(self.get_node(payload, &mut NoOp, 0)?, acc)
};
match node {
Node::Extension(_, ref payload) => handle_payload(payload)?,
Node::Branch(ref payloads, _) => for payload in payloads { handle_payload(payload)? },
_ => {},
}
Ok(())
}
/// Get the root node's RLP.
fn root_node<R: Recorder>(&self, r: &mut R) -> super::Result<Node> {
self.root_data(r).map(|d| Node::decoded(&d))
}
/// Get the data of the root node. /// Get the data of the root node.
fn root_data<R: Recorder>(&self, r: &mut R) -> super::Result<DBValue> { fn root_data(&self) -> super::Result<DBValue> {
self.db.get(self.root).ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) self.db.get(self.root).ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root)))
.map(|node| { r.record(self.root, &*node, 0); node })
} }
/// Get the root node as a `Node`. /// Indentation helper for `format_all`.
fn get_node<'a, R: 'a + Recorder>(&'db self, node: &'db [u8], r: &'a mut R, depth: u32) -> super::Result<Node> {
self.get_raw_or_lookup(node, r, depth).map(|n| Node::decoded(&n))
}
/// Indentation helper for `formal_all`.
fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
for _ in 0..size { for _ in 0..size {
write!(f, " ")?; write!(f, " ")?;
@ -157,8 +92,8 @@ impl<'db> TrieDB<'db> {
Node::Leaf(slice, value) => writeln!(f, "'{:?}: {:?}.", slice, value.pretty())?, Node::Leaf(slice, value) => writeln!(f, "'{:?}: {:?}.", slice, value.pretty())?,
Node::Extension(ref slice, ref item) => { Node::Extension(ref slice, ref item) => {
write!(f, "'{:?} ", slice)?; write!(f, "'{:?} ", slice)?;
if let Ok(node) = self.get_node(&*item, &mut NoOp, 0) { if let Ok(node) = self.get_raw_or_lookup(&*item) {
self.fmt_all(node, f, deepness)?; self.fmt_all(Node::decoded(&node), f, deepness)?;
} }
}, },
Node::Branch(ref nodes, ref value) => { Node::Branch(ref nodes, ref value) => {
@ -168,7 +103,8 @@ impl<'db> TrieDB<'db> {
writeln!(f, "=: {:?}", v.pretty())? writeln!(f, "=: {:?}", v.pretty())?
} }
for i in 0..16 { for i in 0..16 {
match self.get_node(&*nodes[i], &mut NoOp, 0) { let node = self.get_raw_or_lookup(&*nodes[i]);
match node.as_ref().map(|n| Node::decoded(&*n)) {
Ok(Node::Empty) => {}, Ok(Node::Empty) => {},
Ok(n) => { Ok(n) => {
self.fmt_indent(f, deepness + 1)?; self.fmt_indent(f, deepness + 1)?;
@ -189,64 +125,49 @@ impl<'db> TrieDB<'db> {
Ok(()) Ok(())
} }
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<DBValue>>
where 'db: 'key, R: Recorder
{
let root_rlp = self.root_data(r)?;
self.get_from_node(&root_rlp, key, r, 1)
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
/// value exists for the key.
///
/// Note: Not a public API; use Trie trait functions.
fn get_from_node<'key, R: 'key>(
&'db self,
node: &'db [u8],
key: &NibbleSlice<'key>,
r: &'key mut R,
d: u32
) -> super::Result<Option<DBValue>> where 'db: 'key, R: Recorder {
match Node::decoded(node) {
Node::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == *key => Ok(Some(value.clone())),
Node::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) {
let data = self.get_raw_or_lookup(&*item, r, d)?;
self.get_from_node(&data, &key.mid(slice.len()), r, d + 1)
} else {
Ok(None)
}
},
Node::Branch(ref nodes, ref value) => match key.is_empty() {
true => Ok(value.clone()),
false => {
let node = self.get_raw_or_lookup(&*nodes[key.at(0) as usize], r, d)?;
self.get_from_node(&node, &key.mid(1), r, d + 1)
}
},
_ => Ok(None)
}
}
/// Given some node-describing data `node`, return the actual node RLP. /// Given some node-describing data `node`, return the actual node RLP.
/// This could be a simple identity operation in the case that the node is sufficiently small, but /// This could be a simple identity operation in the case that the node is sufficiently small, but
/// may require a database lookup. /// may require a database lookup.
fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<DBValue> { fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result<DBValue> {
// check if its sha3 + len // check if its sha3 + len
let r = Rlp::new(node); let r = Rlp::new(node);
match r.is_data() && r.size() == 32 { match r.is_data() && r.size() == 32 {
true => { true => {
let key = r.as_val::<H256>(); let key = r.as_val::<H256>();
self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
.map(|raw| { rec.record(&key, &raw, d); raw })
} }
false => Ok(DBValue::from_slice(node)) false => Ok(DBValue::from_slice(node))
} }
} }
} }
impl<'db> Trie for TrieDB<'db> {
fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>)
}
fn root(&self) -> &H256 { self.root }
fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
where 'a: 'key
{
Lookup {
db: self.db,
query: query,
hash: self.root.clone(),
}.look_up(NibbleSlice::new(key))
}
}
impl<'db> fmt::Debug for TrieDB<'db> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
writeln!(f, "c={:?} [", self.hash_count)?;
let root_rlp = self.db.get(self.root).expect("Trie root not found!");
self.fmt_all(Node::decoded(&root_rlp), f, 0)?;
writeln!(f, "]")
}
}
#[derive(Clone, Eq, PartialEq)] #[derive(Clone, Eq, PartialEq)]
enum Status { enum Status {
Entering, Entering,
@ -257,7 +178,7 @@ enum Status {
#[derive(Clone, Eq, PartialEq)] #[derive(Clone, Eq, PartialEq)]
struct Crumb { struct Crumb {
node: Node, node: OwnedNode,
status: Status, status: Status,
} }
@ -265,10 +186,10 @@ impl Crumb {
/// Move on to next status in the node's sequence. /// Move on to next status in the node's sequence.
fn increment(&mut self) { fn increment(&mut self) {
self.status = match (&self.status, &self.node) { self.status = match (&self.status, &self.node) {
(_, &Node::Empty) => Status::Exiting, (_, &OwnedNode::Empty) => Status::Exiting,
(&Status::Entering, _) => Status::At, (&Status::Entering, _) => Status::At,
(&Status::At, &Node::Branch(_, _)) => Status::AtChild(0), (&Status::At, &OwnedNode::Branch(_, _)) => Status::AtChild(0),
(&Status::AtChild(x), &Node::Branch(_, _)) if x < 15 => Status::AtChild(x + 1), (&Status::AtChild(x), &OwnedNode::Branch(_, _)) if x < 15 => Status::AtChild(x + 1),
_ => Status::Exiting, _ => Status::Exiting,
} }
} }
@ -291,41 +212,40 @@ impl<'a> TrieDBIterator<'a> {
key_nibbles: Vec::new(), key_nibbles: Vec::new(),
}; };
db.root_data(&mut NoOp).and_then(|root| r.descend(&root))?; db.root_data().and_then(|root| r.descend(&root))?;
Ok(r) Ok(r)
} }
fn seek_descend<'key> ( &mut self, node: &[u8], key: &NibbleSlice<'key>, d: u32) -> super::Result<()> { fn seek_descend<'key>(&mut self, node_data: DBValue, key: &NibbleSlice<'key>) -> super::Result<()> {
match Node::decoded(node) { let node = Node::decoded(&node_data);
match node {
Node::Leaf(ref slice, _) => { Node::Leaf(ref slice, _) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if slice == key { if slice == key {
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::At, status: Status::At,
node: Node::decoded(node), node: node.clone().into(),
}); });
} else { } else {
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::Exiting, status: Status::Exiting,
node: Node::decoded(node), node: node.clone().into(),
}); });
} }
self.key_nibbles.extend(slice.iter()); self.key_nibbles.extend(slice.iter());
Ok(()) Ok(())
}, },
Node::Extension(ref slice, ref item) => { Node::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) { if key.starts_with(slice) {
let mut r = NoOp;
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::At, status: Status::At,
node: Node::decoded(node), node: node.clone().into(),
}); });
self.key_nibbles.extend(slice.iter()); self.key_nibbles.extend(slice.iter());
let data = self.db.get_raw_or_lookup(&*item, &mut r, d)?; let data = self.db.get_raw_or_lookup(&*item)?;
self.seek_descend(&data, &key.mid(slice.len()), d + 1) self.seek_descend(data, &key.mid(slice.len()))
} else { } else {
self.descend(node)?; self.descend(&node_data)?;
Ok(()) Ok(())
} }
}, },
@ -333,20 +253,19 @@ impl<'a> TrieDBIterator<'a> {
true => { true => {
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::At, status: Status::At,
node: Node::decoded(node), node: node.clone().into(),
}); });
Ok(()) Ok(())
}, },
false => { false => {
let mut r = NoOp;
let i = key.at(0); let i = key.at(0);
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::AtChild(i as usize), status: Status::AtChild(i as usize),
node: Node::decoded(node), node: node.clone().into(),
}); });
self.key_nibbles.push(i); self.key_nibbles.push(i);
let child = self.db.get_raw_or_lookup(&*nodes[i as usize], &mut r, d)?; let child = self.db.get_raw_or_lookup(&*nodes[i as usize])?;
self.seek_descend(&child, &key.mid(1), d + 1) self.seek_descend(child, &key.mid(1))
} }
}, },
_ => Ok(()) _ => Ok(())
@ -357,10 +276,12 @@ impl<'a> TrieDBIterator<'a> {
fn descend(&mut self, d: &[u8]) -> super::Result<()> { fn descend(&mut self, d: &[u8]) -> super::Result<()> {
self.trail.push(Crumb { self.trail.push(Crumb {
status: Status::Entering, status: Status::Entering,
node: self.db.get_node(d, &mut NoOp, 0)?, node: Node::decoded(&self.db.get_raw_or_lookup(d)?).into(),
}); });
match self.trail.last().expect("just pushed item; qed").node { match &self.trail.last().expect("just pushed item; qed").node {
Node::Leaf(ref n, _) | Node::Extension(ref n, _) => { self.key_nibbles.extend(NibbleSlice::from_encoded(n).0.iter()); }, &OwnedNode::Leaf(ref n, _) | &OwnedNode::Extension(ref n, _) => {
self.key_nibbles.extend((0..n.len()).map(|i| n.at(i)));
},
_ => {} _ => {}
} }
@ -379,9 +300,8 @@ impl<'a> TrieIterator for TrieDBIterator<'a> {
fn seek(&mut self, key: &[u8]) -> super::Result<()> { fn seek(&mut self, key: &[u8]) -> super::Result<()> {
self.trail.clear(); self.trail.clear();
self.key_nibbles.clear(); self.key_nibbles.clear();
let mut r = NoOp; let root_rlp = self.db.root_data()?;
let root_rlp = self.db.root_data(&mut r)?; self.seek_descend(root_rlp, &NibbleSlice::new(key))
self.seek_descend(&root_rlp, &NibbleSlice::new(key), 1)
} }
} }
@ -397,27 +317,27 @@ impl<'a> Iterator for TrieDBIterator<'a> {
match (b.status, b.node) { match (b.status, b.node) {
(Status::Exiting, n) => { (Status::Exiting, n) => {
match n { match n {
Node::Leaf(n, _) | Node::Extension(n, _) => { OwnedNode::Leaf(n, _) | OwnedNode::Extension(n, _) => {
let l = self.key_nibbles.len(); let l = self.key_nibbles.len();
self.key_nibbles.truncate(l - NibbleSlice::from_encoded(&*n).0.len()); self.key_nibbles.truncate(l - n.len());
}, },
Node::Branch(_, _) => { self.key_nibbles.pop(); }, OwnedNode::Branch(_, _) => { self.key_nibbles.pop(); },
_ => {} _ => {}
} }
self.trail.pop(); self.trail.pop();
// continue // continue
}, },
(Status::At, Node::Leaf(_, v)) | (Status::At, Node::Branch(_, Some(v))) => { (Status::At, OwnedNode::Leaf(_, v)) | (Status::At, OwnedNode::Branch(_, Some(v))) => {
return Some(Ok((self.key(), v))); return Some(Ok((self.key(), v)));
}, },
(Status::At, Node::Extension(_, d)) => { (Status::At, OwnedNode::Extension(_, d)) => {
if let Err(e) = self.descend(&*d) { if let Err(e) = self.descend(&*d) {
return Some(Err(e)); return Some(Err(e));
} }
// continue // continue
}, },
(Status::At, Node::Branch(_, _)) => {}, (Status::At, OwnedNode::Branch(_, _)) => {},
(Status::AtChild(i), Node::Branch(ref children, _)) if children[i].len() > 0 => { (Status::AtChild(i), OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => {
match i { match i {
0 => self.key_nibbles.push(0), 0 => self.key_nibbles.push(0),
i => *self.key_nibbles.last_mut() i => *self.key_nibbles.last_mut()
@ -428,7 +348,7 @@ impl<'a> Iterator for TrieDBIterator<'a> {
} }
// continue // continue
}, },
(Status::AtChild(i), Node::Branch(_, _)) => { (Status::AtChild(i), OwnedNode::Branch(_, _)) => {
if i == 0 { if i == 0 {
self.key_nibbles.push(0); self.key_nibbles.push(0);
} }
@ -440,29 +360,6 @@ impl<'a> Iterator for TrieDBIterator<'a> {
} }
} }
impl<'db> Trie for TrieDB<'db> {
fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>)
}
fn root(&self) -> &H256 { self.root }
fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
where 'a: 'b, R: Recorder
{
self.do_lookup(&NibbleSlice::new(key), rec)
}
}
impl<'db> fmt::Debug for TrieDB<'db> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
writeln!(f, "c={:?} [", self.hash_count)?;
let root_rlp = self.db.get(self.root).expect("Trie root not found!");
self.fmt_all(Node::decoded(&root_rlp), f, 0)?;
writeln!(f, "]")
}
}
#[test] #[test]
fn iterator() { fn iterator() {
use memorydb::*; use memorydb::*;
@ -529,3 +426,23 @@ fn iterator_seek() {
iter.seek(b"C").unwrap(); iter.seek(b"C").unwrap();
assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]); assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
} }
#[test]
fn get_len() {
use memorydb::*;
use super::TrieMut;
use super::triedbmut::*;
let mut memdb = MemoryDB::new();
let mut root = H256::new();
{
let mut t = TrieDBMut::new(&mut memdb, &mut root);
t.insert(b"A", b"ABC").unwrap();
t.insert(b"B", b"ABCBA").unwrap();
}
let t = TrieDB::new(&memdb, &root).unwrap();
assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()), Ok(Some(3)));
assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()), Ok(Some(5)));
assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None));
}

View File

@ -17,6 +17,7 @@
//! In-memory trie representation. //! In-memory trie representation.
use super::{TrieError, TrieMut}; use super::{TrieError, TrieMut};
use super::lookup::Lookup;
use super::node::Node as RlpNode; use super::node::Node as RlpNode;
use super::node::NodeKey; use super::node::NodeKey;
@ -100,22 +101,22 @@ impl Node {
fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self {
match RlpNode::decoded(rlp) { match RlpNode::decoded(rlp) {
RlpNode::Empty => Node::Empty, RlpNode::Empty => Node::Empty,
RlpNode::Leaf(k, v) => Node::Leaf(k, v), RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)),
RlpNode::Extension(key, cb) => { RlpNode::Extension(key, cb) => {
Node::Extension(key, Self::inline_or_hash(&*cb, db, storage)) Node::Extension(key.encoded(false), Self::inline_or_hash(cb, db, storage))
} }
RlpNode::Branch(children_rlp, val) => { RlpNode::Branch(children_rlp, val) => {
let mut children = empty_children(); let mut children = empty_children();
for i in 0..16 { for i in 0..16 {
let raw = &children_rlp[i]; let raw = children_rlp[i];
let child_rlp = Rlp::new(&*raw); let child_rlp = Rlp::new(raw);
if !child_rlp.is_empty() { if !child_rlp.is_empty() {
children[i] = Some(Self::inline_or_hash(&*raw, db, storage)); children[i] = Some(Self::inline_or_hash(raw, db, storage));
} }
} }
Node::Branch(children, val) Node::Branch(children, val.map(DBValue::from_slice))
} }
} }
} }
@ -370,7 +371,11 @@ impl<'a> TrieDBMut<'a> {
where 'x: 'key where 'x: 'key
{ {
match *handle { match *handle {
NodeHandle::Hash(ref hash) => self.do_db_lookup(hash, partial), NodeHandle::Hash(ref hash) => Lookup {
db: &*self.db,
query: DBValue::from_slice,
hash: hash.clone(),
}.look_up(partial),
NodeHandle::InMemory(ref handle) => match self.storage[handle] { NodeHandle::InMemory(ref handle) => match self.storage[handle] {
Node::Empty => Ok(None), Node::Empty => Ok(None),
Node::Leaf(ref key, ref value) => { Node::Leaf(ref key, ref value) => {
@ -403,54 +408,6 @@ impl<'a> TrieDBMut<'a> {
} }
} }
/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
where 'x: 'key
{
self.db.get(hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(*hash)))
.and_then(|node_rlp| self.get_from_db_node(&node_rlp, key))
}
/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
/// value exists for the key.
///
/// Note: Not a public API; use Trie trait functions.
fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
where 'x: 'key
{
match RlpNode::decoded(node) {
RlpNode::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == key => Ok(Some(value.clone())),
RlpNode::Extension(ref slice, ref item) => {
let slice = &NibbleSlice::from_encoded(slice).0;
if key.starts_with(slice) {
self.get_from_db_node(&self.get_raw_or_lookup(&*item)?, key.mid(slice.len()))
} else {
Ok(None)
}
},
RlpNode::Branch(ref nodes, ref value) => match key.is_empty() {
true => Ok(value.clone()),
false => self.get_from_db_node(&self.get_raw_or_lookup(&*nodes[key.at(0) as usize])?, key.mid(1))
},
_ => Ok(None),
}
}
/// Given some node-describing data `node`, return the actual node RLP.
/// This could be a simple identity operation in the case that the node is sufficiently small, but
/// may require a database lookup.
fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<DBValue> {
// check if its sha3 + len
let r = Rlp::new(node);
match r.is_data() && r.size() == 32 {
true => {
let key = r.as_val::<H256>();
self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
}
false => Ok(DBValue::from_slice(node))
}
}
/// insert a key, value pair into the trie, creating new nodes if necessary. /// insert a key, value pair into the trie, creating new nodes if necessary.
fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option<DBValue>) fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option<DBValue>)
-> super::Result<(StorageHandle, bool)> -> super::Result<(StorageHandle, bool)>