Zero-alloc trie lookups (#3998)

* triedb cleanup

* factor out common portion of trie query

* allocate far fewer times in node decoding

* fix bench compilation

* introduce OwnedNode variant to make iter fast again

* generalize recorder trait to Query

* decode trie outputs cost-free in state

* test for passing closure as query

ethcore/src/state/account.rs

@@ -178,8 +178,8 @@ impl Account {
 			SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \
 			using it will not fail.");
 
-		let item: U256 = match db.get(key){
+		let item: U256 = match db.get_with(key, ::rlp::decode) {
-			Ok(x) => x.map_or_else(U256::zero, |v| decode(&*v)),
+			Ok(x) => x.unwrap_or_else(U256::zero),
 			Err(e) => panic!("Encountered potential DB corruption: {}", e),
 		};
 		let value: H256 = item.into();
@@ -453,12 +453,12 @@ impl Account {
 	/// omitted.
 	pub fn prove_storage(&self, db: &HashDB, storage_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> {
 		use util::trie::{Trie, TrieDB};
-		use util::trie::recorder::{Recorder, BasicRecorder as TrieRecorder};
+		use util::trie::recorder::Recorder;
 
-		let mut recorder = TrieRecorder::with_depth(from_level);
+		let mut recorder = Recorder::with_depth(from_level);
 
 		let trie = TrieDB::new(db, &self.storage_root)?;
-		let _ = trie.get_recorded(&storage_key, &mut recorder)?;
+		let _ = trie.get_with(&storage_key, &mut recorder)?;
 
 		Ok(recorder.drain().into_iter().map(|r| r.data).collect())
 	}

ethcore/src/state/mod.rs

@@ -32,7 +32,7 @@ use state_db::StateDB;
 
 use util::*;
 
-use util::trie::recorder::{Recorder, BasicRecorder as TrieRecorder};
+use util::trie::recorder::Recorder;
 
 mod account;
 mod substate;
@@ -425,8 +425,8 @@ impl State {
 
 		// account is not found in the global cache, get from the DB and insert into local
 		let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
-		let maybe_acc = match db.get(address) {
+		let maybe_acc = match db.get_with(address, Account::from_rlp) {
-			Ok(acc) => acc.map(|v| Account::from_rlp(&v)),
+			Ok(acc) => acc,
 			Err(e) => panic!("Potential DB corruption encountered: {}", e),
 		};
 		let r = maybe_acc.as_ref().map_or(H256::new(), |a| {
@@ -690,8 +690,8 @@ impl State {
 
 				// not found in the global cache, get from the DB and insert into local
 				let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
-				let mut maybe_acc = match db.get(a) {
+				let mut maybe_acc = match db.get_with(a, Account::from_rlp) {
-					Ok(acc) => acc.map(|v| Account::from_rlp(&v)),
+					Ok(acc) => acc,
 					Err(e) => panic!("Potential DB corruption encountered: {}", e),
 				};
 				if let Some(ref mut account) = maybe_acc.as_mut() {
@@ -722,9 +722,8 @@ impl State {
 				None => {
 					let maybe_acc = if self.db.check_non_null_bloom(a) {
 						let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
-						match db.get(a) {
+						match db.get_with(a, Account::from_rlp) {
-							Ok(Some(acc)) => AccountEntry::new_clean(Some(Account::from_rlp(&acc))),
+							Ok(acc) => AccountEntry::new_clean(acc),
-							Ok(None) => AccountEntry::new_clean(None),
 							Err(e) => panic!("Potential DB corruption encountered: {}", e),
 						}
 					} else {
@@ -770,9 +769,9 @@ impl State {
 	/// Requires a secure trie to be used for accurate results.
 	/// `account_key` == sha3(address)
 	pub fn prove_account(&self, account_key: H256, from_level: u32) -> Result<Vec<Bytes>, Box<TrieError>> {
-		let mut recorder = TrieRecorder::with_depth(from_level);
+		let mut recorder = Recorder::with_depth(from_level);
 		let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
-		let _  = trie.get_recorded(&account_key, &mut recorder)?;
+		trie.get_with(&account_key, &mut recorder)?;
 
 		Ok(recorder.drain().into_iter().map(|r| r.data).collect())
 	}
@@ -786,8 +785,8 @@ impl State {
 		// TODO: probably could look into cache somehow but it's keyed by
 		// address, not sha3(address).
 		let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
-		let acc = match trie.get(&account_key)? {
+		let acc = match trie.get_with(&account_key, Account::from_rlp)? {
-			Some(rlp) => Account::from_rlp(&rlp),
+			Some(acc) => acc,
 			None => return Ok(Vec::new()),
 		};
 
@@ -799,8 +798,8 @@ impl State {
 	/// Only works when backed by a secure trie.
 	pub fn code_by_address_hash(&self, account_key: H256) -> Result<Option<Bytes>, Box<TrieError>> {
 		let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?;
-		let mut acc = match trie.get(&account_key)? {
+		let mut acc = match trie.get_with(&account_key, Account::from_rlp)? {
-			Some(rlp) => Account::from_rlp(&rlp),
+			Some(acc) => acc,
 			None => return Ok(None),
 		};
 

util/benches/bigint.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-//! benchmarking for rlp
+//! benchmarking for bigint
 //! should be started with:
 //! ```bash
 //! multirust run nightly cargo bench
@@ -24,10 +24,10 @@
 #![feature(asm)]
 
 extern crate test;
-extern crate ethcore_bigint as bigint;
+extern crate ethcore_util;
 
 use test::{Bencher, black_box};
-use bigint::uint::{U256, U512, Uint, U128};
+use ethcore_util::{U256, U512, Uint, U128};
 
 #[bench]
 fn u256_add(b: &mut Bencher) {

util/benches/rlp.rs

@@ -24,12 +24,12 @@
 
 extern crate test;
 extern crate rlp;
-extern crate ethcore_bigint as bigint;
+extern crate ethcore_util as util;
 
 use test::Bencher;
 use std::str::FromStr;
 use rlp::*;
-use bigint::uint::U256;
+use util::U256;
 
 #[bench]
 fn bench_stream_u64_value(b: &mut Bencher) {

util/src/nibblevec.rs

@@ -18,27 +18,26 @@
 //! An owning, nibble-oriented byte vector.
 
 use ::NibbleSlice;
+use elastic_array::ElasticArray36;
 
-#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Debug)]
 /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`.
+#[derive(Clone, PartialEq, Eq, Debug)]
 pub struct NibbleVec {
-	inner: Vec<u8>,
+	inner: ElasticArray36<u8>,
 	len: usize,
 }
 
+impl Default for NibbleVec {
+	fn default() -> Self {
+		NibbleVec::new()
+	}
+}
+
 impl NibbleVec {
 	/// Make a new `NibbleVec`
 	pub fn new() -> Self {
 		NibbleVec {
-			inner: Vec::new(),
+			inner: ElasticArray36::new(),
-			len: 0
-		}
-	}
-
-	/// Make a `NibbleVec` with capacity for `n` nibbles.
-	pub fn with_capacity(n: usize) -> Self {
-		NibbleVec {
-			inner: Vec::with_capacity((n / 2) + (n % 2)),
 			len: 0
 		}
 	}
@@ -49,9 +48,6 @@ impl NibbleVec {
 	/// Retrurns true if `NibbleVec` has zero length
 	pub fn is_empty(&self) -> bool { self.len == 0 }
 
-	/// Capacity of the `NibbleVec`.
-	pub fn capacity(&self) -> usize { self.inner.capacity() * 2 }
-
 	/// Try to get the nibble at the given offset.
 	pub fn at(&self, idx: usize) -> u8 {
 		if idx % 2 == 0 {
@@ -109,7 +105,7 @@ impl NibbleVec {
 
 impl<'a> From<NibbleSlice<'a>> for NibbleVec {
 	fn from(s: NibbleSlice<'a>) -> Self {
-		let mut v = NibbleVec::with_capacity(s.len());
+		let mut v = NibbleVec::new();
 		for i in 0..s.len() {
 			v.push(s.at(i));
 		}

util/src/trie/fatdb.rs

@@ -16,8 +16,8 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::{HashDB, DBValue};
+use hashdb::HashDB;
-use super::{TrieDB, Trie, TrieDBIterator, TrieItem, Recorder, TrieIterator};
+use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query};
 
 /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
 /// Additionaly it stores inserted hash-key mappings for later retrieval.
@@ -58,10 +58,10 @@ impl<'db> Trie for FatDB<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
+	fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
-		where 'a: 'b, R: Recorder
+		where 'a: 'key
 	{
-		self.raw.get_recorded(&key.sha3(), rec)
+		self.raw.get_with(&key.sha3(), query)
 	}
 }
 
@@ -104,6 +104,7 @@ impl<'db> Iterator for FatDBIterator<'db> {
 #[test]
 fn fatdb_to_trie() {
 	use memorydb::MemoryDB;
+	use hashdb::DBValue;
 	use trie::{FatDBMut, TrieMut};
 
 	let mut memdb = MemoryDB::new();

util/src/trie/lookup.rs

@@ -0,0 +1,94 @@
+// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Trie lookup via HashDB.
+
+use hashdb::HashDB;
+use nibbleslice::NibbleSlice;
+use rlp::{Rlp, View};
+use ::{H256};
+
+use super::{TrieError, Query};
+use super::node::Node;
+
+/// Trie lookup helper object.
+pub struct Lookup<'a, Q: Query> {
+	/// database to query from.
+	pub db: &'a HashDB,
+	/// Query object to record nodes and transform data.
+	pub query: Q,
+	/// Hash to start at
+	pub hash: H256,
+}
+
+impl<'a, Q: Query> Lookup<'a, Q> {
+	/// Look up the given key. If the value is found, it will be passed to the given
+	/// function to decode or copy.
+	pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result<Option<Q::Item>> {
+		let mut hash = self.hash;
+
+		// this loop iterates through non-inline nodes.
+		for depth in 0.. {
+			let node_data = match self.db.get(&hash) {
+				Some(value) => value,
+				None => return Err(Box::new(match depth {
+					0 => TrieError::InvalidStateRoot(hash),
+					_ => TrieError::IncompleteDatabase(hash),
+				})),
+			};
+
+			self.query.record(&hash, &node_data, depth);
+
+			// this loop iterates through all inline children (usually max 1)
+			// without incrementing the depth.
+			let mut node_data = &node_data[..];
+			loop {
+				match Node::decoded(node_data) {
+					Node::Leaf(slice, value) => {
+						return Ok(match slice == key {
+							true => Some(self.query.decode(value)),
+							false => None,
+						})
+					}
+					Node::Extension(slice, item) => {
+						if key.starts_with(&slice) {
+							node_data = item;
+							key = key.mid(slice.len());
+						} else {
+							return Ok(None)
+						}
+					}
+					Node::Branch(children, value) => match key.is_empty() {
+						true => return Ok(value.map(move |val| self.query.decode(val))),
+						false => {
+							node_data = children[key.at(0) as usize];
+							key = key.mid(1);
+						}
+					},
+					_ => return Ok(None),
+				}
+
+				// check if new node data is inline or hash.
+				let r = Rlp::new(node_data);
+				if r.is_data() && r.size() == 32 {
+					hash = r.as_val();
+					break
+				}
+			}
+		}
+		Ok(None)
+	}
+}

util/src/trie/mod.rs

@@ -38,6 +38,7 @@ pub mod recorder;
 
 mod fatdb;
 mod fatdbmut;
+mod lookup;
 
 pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
 pub use self::triedbmut::TrieDBMut;
@@ -76,6 +77,46 @@ pub type Result<T> = ::std::result::Result<T, Box<TrieError>>;
 /// Trie-Item type.
 pub type TrieItem<'a> = Result<(Vec<u8>, DBValue)>;
 
+/// Description of what kind of query will be made to the trie.
+///
+/// This is implemented for any &mut recorder (where the query will return
+/// a DBValue), any function taking raw bytes (where no recording will be made),
+/// or any tuple of (&mut Recorder, FnOnce(&[u8]))
+pub trait Query {
+	/// Output item.
+	type Item;
+
+	/// Decode a byte-slice into the desired item.
+	fn decode(self, &[u8]) -> Self::Item;
+
+	/// Record that a node has been passed through.
+	fn record(&mut self, &H256, &[u8], u32) { }
+}
+
+impl<'a> Query for &'a mut Recorder {
+	type Item = DBValue;
+
+	fn decode(self, value: &[u8]) -> DBValue { DBValue::from_slice(value) }
+	fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
+		(&mut **self).record(hash, data, depth);
+	}
+}
+
+impl<F, T> Query for F where F: for<'a> FnOnce(&'a [u8]) -> T {
+	type Item = T;
+
+	fn decode(self, value: &[u8]) -> T { (self)(value) }
+}
+
+impl<'a, F, T> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T {
+	type Item = T;
+
+	fn decode(self, value: &[u8]) -> T { (self.1)(value) }
+	fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
+		self.0.record(hash, data, depth)
+	}
+}
+
 /// A key-value datastore implemented as a database-backed modified Merkle tree.
 pub trait Trie {
 	/// Return the root of the trie.
@@ -91,13 +132,13 @@ pub trait Trie {
 
 	/// What is the value of the given key in this trie?
 	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result<Option<DBValue>> where 'a: 'key {
-		self.get_recorded(key, &mut recorder::NoOp)
+		self.get_with(key, DBValue::from_slice)
 	}
 
-	/// Query the value of the given key in this trie while recording visited nodes
+	/// Search for the key with the given query parameter. See the docs of the `Query`
-	/// to the given recorder. If the query encounters an error, the nodes passed to the recorder are unspecified.
+	/// trait for more details.
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> Result<Option<DBValue>>
+	fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q)
-		where 'a: 'b, R: Recorder;
+		-> Result<Option<Q::Item>> where 'a: 'key;
 
 	/// Returns a depth-first iterator over the elements of trie.
 	fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>>;
@@ -192,9 +233,10 @@ impl<'db> Trie for TrieKinds<'db> {
 		wrapper!(self, contains, key)
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], r: &'b mut R) -> Result<Option<DBValue>>
+	fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result<Option<Q::Item>>
-		where 'a: 'b, R: Recorder {
+		where 'a: 'key
-		wrapper!(self, get_recorded, key, r)
+	{
+		wrapper!(self, get_with, key, query)
 	}
 
 	fn iter<'a>(&'a self) -> Result<Box<TrieIterator<Item = TrieItem> + 'a>> {

util/src/trie/node.rs

@@ -16,6 +16,7 @@
 
 use elastic_array::ElasticArray36;
 use nibbleslice::*;
+use nibblevec::NibbleVec;
 use bytes::*;
 use rlp::*;
 use hashdb::DBValue;
@@ -24,40 +25,21 @@ use hashdb::DBValue;
 pub type NodeKey = ElasticArray36<u8>;
 
 /// Type of node in the trie and essential information thereof.
-#[derive(Eq, PartialEq, Debug)]
+#[derive(Eq, PartialEq, Debug, Clone)]
-pub enum Node {
+pub enum Node<'a> {
 	/// Null trie node; could be an empty root or an empty branch entry.
 	Empty,
 	/// Leaf node; has key slice and value. Value may not be empty.
-	Leaf(NodeKey, DBValue),
+	Leaf(NibbleSlice<'a>, &'a [u8]),
 	/// Extension node; has key slice and node data. Data may not be null.
-	Extension(NodeKey, DBValue),
+	Extension(NibbleSlice<'a>, &'a [u8]),
 	/// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data.
-	Branch([NodeKey; 16], Option<DBValue>)
+	Branch([&'a [u8]; 16], Option<&'a [u8]>)
 }
 
-impl Clone for Node {
+impl<'a> Node<'a> {
-	fn clone(&self) -> Node {
-		match *self {
-			Node::Empty => Node::Empty,
-			Node::Leaf(ref k, ref v) => Node::Leaf(k.clone(), v.clone()),
-			Node::Extension(ref k, ref v) => Node::Extension(k.clone(), v.clone()),
-			Node::Branch(ref k, ref v) => {
-				let mut branch = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
-					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
-					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
-				for i in 0 .. 16 {
-					branch[i] = k[i].clone();
-				}
-				Node::Branch(branch, v.clone())
-			}
-		}
-	}
-}
-
-impl Node {
 	/// Decode the `node_rlp` and return the Node.
-	pub fn decoded(node_rlp: &[u8]) -> Node {
+	pub fn decoded(node_rlp: &'a [u8]) -> Self {
 		let r = Rlp::new(node_rlp);
 		match r.prototype() {
 			// either leaf or extension - decode first item with NibbleSlice::???
@@ -66,18 +48,16 @@ impl Node {
 			// if extension, second item is a node (either SHA3 to be looked up and
 			// fed back into this function or inline RLP which can be fed back into this function).
 			Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) {
-				(slice, true) => Node::Leaf(slice.encoded(true), DBValue::from_slice(r.at(1).data())),
+				(slice, true) => Node::Leaf(slice, r.at(1).data()),
-				(slice, false) => Node::Extension(slice.encoded(false), DBValue::from_slice(r.at(1).as_raw())),
+				(slice, false) => Node::Extension(slice, r.at(1).as_raw()),
 			},
 			// branch - first 16 are nodes, 17th is a value (or empty).
 			Prototype::List(17) => {
-				let mut nodes: [NodeKey; 16] = [NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+				let mut nodes = [&[] as &[u8]; 16];
-					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
-					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new()];
 				for i in 0..16 {
-					nodes[i] = NodeKey::from_slice(r.at(i).as_raw());
+					nodes[i] = r.at(i).as_raw();
 				}
-				Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(DBValue::from_slice(r.at(16).data())) })
+				Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) })
 			},
 			// an empty branch index.
 			Prototype::Data(0) => Node::Empty,
@@ -94,23 +74,23 @@ impl Node {
 		match *self {
 			Node::Leaf(ref slice, ref value) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&&**slice);
+				stream.append(&&*slice.encoded(true));
-				stream.append(&&**value);
+				stream.append(value);
 				stream.out()
 			},
 			Node::Extension(ref slice, ref raw_rlp) => {
 				let mut stream = RlpStream::new_list(2);
-				stream.append(&&**slice);
+				stream.append(&&*slice.encoded(false));
-				stream.append_raw(&&*raw_rlp, 1);
+				stream.append_raw(raw_rlp, 1);
 				stream.out()
 			},
 			Node::Branch(ref nodes, ref value) => {
 				let mut stream = RlpStream::new_list(17);
 				for i in 0..16 {
-					stream.append_raw(&*nodes[i], 1);
+					stream.append_raw(nodes[i], 1);
 				}
 				match *value {
-					Some(ref n) => { stream.append(&&**n); },
+					Some(ref n) => { stream.append(n); },
 					None => { stream.append_empty_data(); },
 				}
 				stream.out()
@@ -123,3 +103,64 @@ impl Node {
 		}
 	}
 }
+
+/// An owning node type. Useful for trie iterators.
+#[derive(Debug, PartialEq, Eq)]
+pub enum OwnedNode {
+	/// Empty trie node.
+	Empty,
+	/// Leaf node: partial key and value.
+	Leaf(NibbleVec, DBValue),
+	/// Extension node: partial key and child node.
+	Extension(NibbleVec, DBValue),
+	/// Branch node: 16 children and an optional value.
+	Branch([NodeKey; 16], Option<DBValue>),
+}
+
+impl Clone for OwnedNode {
+	fn clone(&self) -> Self {
+		match *self {
+			OwnedNode::Empty => OwnedNode::Empty,
+			OwnedNode::Leaf(ref k, ref v) => OwnedNode::Leaf(k.clone(), v.clone()),
+			OwnedNode::Extension(ref k, ref c) => OwnedNode::Extension(k.clone(), c.clone()),
+			OwnedNode::Branch(ref c, ref v) => {
+				let mut children = [
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+				];
+
+				for (owned, borrowed) in children.iter_mut().zip(c.iter()) {
+					*owned = borrowed.clone()
+				}
+
+				OwnedNode::Branch(children, v.as_ref().cloned())
+			}
+		}
+	}
+}
+
+impl<'a> From<Node<'a>> for OwnedNode {
+	fn from(node: Node<'a>) -> Self {
+		match node {
+			Node::Empty => OwnedNode::Empty,
+			Node::Leaf(k, v) => OwnedNode::Leaf(k.into(), DBValue::from_slice(v)),
+			Node::Extension(k, child) => OwnedNode::Extension(k.into(), DBValue::from_slice(child)),
+			Node::Branch(c, val) => {
+				let mut children = [
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+					NodeKey::new(), NodeKey::new(), NodeKey::new(), NodeKey::new(),
+				];
+
+				for (owned, borrowed) in children.iter_mut().zip(c.iter()) {
+					*owned = NodeKey::from_slice(borrowed)
+				}
+
+				OwnedNode::Branch(children, val.map(DBValue::from_slice))
+			}
+		}
+	}
+}

util/src/trie/recorder.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+//! Trie query recorder.
+
 use sha3::Hashable;
 use {Bytes, H256};
 
@@ -30,63 +32,36 @@ pub struct Record {
 	pub hash: H256,
 }
 
-/// Trie node recorder.
+/// Records trie nodes as they pass it.
-///
-/// These are used to record which nodes are visited during a trie query.
-/// Inline nodes are not to be recorded, as they are contained within their parent.
-pub trait Recorder {
-	/// Record that the given node has been visited.
-	///
-	/// The depth parameter is the depth of the visited node, with the root node having depth 0.
-	fn record(&mut self, hash: &H256, data: &[u8], depth: u32);
-
-	/// Drain all accepted records from the recorder in ascending order by depth.
-	fn drain(&mut self) -> Vec<Record> where Self: Sized;
-}
-
-/// A no-op trie recorder. This ignores everything which is thrown at it.
-pub struct NoOp;
-
-impl Recorder for NoOp {
-	#[inline]
-	fn record(&mut self, _hash: &H256, _data: &[u8], _depth: u32) {}
-
-	#[inline]
-	fn drain(&mut self) -> Vec<Record> { Vec::new() }
-}
-
-/// A simple recorder. Does nothing fancy but fulfills the `Recorder` interface
-/// properly.
 #[derive(Debug)]
-pub struct BasicRecorder {
+pub struct Recorder {
 	nodes: Vec<Record>,
 	min_depth: u32,
 }
 
-impl Default for BasicRecorder {
+impl Default for Recorder {
 	fn default() -> Self {
-		BasicRecorder::new()
+		Recorder::new()
 	}
 }
 
-impl BasicRecorder {
+impl Recorder {
-	/// Create a new `BasicRecorder` which records all given nodes.
+	/// Create a new `Recorder` which records all given nodes.
 	#[inline]
 	pub fn new() -> Self {
-		BasicRecorder::with_depth(0)
+		Recorder::with_depth(0)
 	}
 
-	/// Create a `BasicRecorder` which only records nodes beyond a given depth.
+	/// Create a `Recorder` which only records nodes beyond a given depth.
 	pub fn with_depth(depth: u32) -> Self {
-		BasicRecorder {
+		Recorder {
 			nodes: Vec::new(),
 			min_depth: depth,
 		}
 	}
-}
 
-impl Recorder for BasicRecorder {
+	/// Record a visited node, given its hash, data, and depth.
-	fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
+	pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) {
 		debug_assert_eq!(data.sha3(), *hash);
 
 		if depth >= self.min_depth {
@@ -98,7 +73,8 @@ impl Recorder for BasicRecorder {
 		}
 	}
 
-	fn drain(&mut self) -> Vec<Record> {
+	/// Drain all visited records.
+	pub fn drain(&mut self) -> Vec<Record> {
 		::std::mem::replace(&mut self.nodes, Vec::new())
 	}
 }
@@ -109,20 +85,9 @@ mod tests {
 	use sha3::Hashable;
 	use ::H256;
 
-	#[test]
-	fn no_op_does_nothing() {
-		let mut no_op = NoOp;
-		let (node1, node2) = (&[1], &[2]);
-		let (hash1, hash2) = (node1.sha3(), node2.sha3());
-		no_op.record(&hash1, node1, 1);
-		no_op.record(&hash2, node2, 2);
-
-		assert_eq!(no_op.drain(), Vec::new());
-	}
-
 	#[test]
 	fn basic_recorder() {
-		let mut basic = BasicRecorder::new();
+		let mut basic = Recorder::new();
 
 		let node1 = vec![1, 2, 3, 4];
 		let node2 = vec![4, 5, 6, 7, 8, 9, 10];
@@ -148,7 +113,7 @@ mod tests {
 
 	#[test]
 	fn basic_recorder_min_depth() {
-		let mut basic = BasicRecorder::with_depth(400);
+		let mut basic = Recorder::with_depth(400);
 
 		let node1 = vec![1, 2, 3, 4];
 		let node2 = vec![4, 5, 6, 7, 8, 9, 10];
@@ -192,9 +157,9 @@ mod tests {
 		}
 
 		let trie = TrieDB::new(&db, &root).unwrap();
-		let mut recorder = BasicRecorder::new();
+		let mut recorder = Recorder::new();
 
-		trie.get_recorded(b"pirate", &mut recorder).unwrap().unwrap();
+		trie.get_with(b"pirate", &mut recorder).unwrap().unwrap();
 
 		let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect();
 		assert_eq!(nodes, vec![
@@ -213,7 +178,7 @@ mod tests {
 			]
 		]);
 
-		trie.get_recorded(b"letter", &mut recorder).unwrap().unwrap();
+		trie.get_with(b"letter", &mut recorder).unwrap().unwrap();
 
 		let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect();
 		assert_eq!(nodes, vec![

util/src/trie/sectriedb.rs

@@ -16,9 +16,9 @@
 
 use hash::H256;
 use sha3::Hashable;
-use hashdb::{HashDB, DBValue};
+use hashdb::HashDB;
 use super::triedb::TrieDB;
-use super::{Trie, TrieItem, Recorder, TrieIterator};
+use super::{Trie, TrieItem, TrieIterator, Query};
 
 /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
 ///
@@ -59,16 +59,17 @@ impl<'db> Trie for SecTrieDB<'db> {
 		self.raw.contains(&key.sha3())
 	}
 
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
+	fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
-		where 'a: 'b, R: Recorder
+		where 'a: 'key
 	{
-		self.raw.get_recorded(&key.sha3(), rec)
+		self.raw.get_with(&key.sha3(), query)
 	}
 }
 
 #[test]
 fn trie_to_sectrie() {
 	use memorydb::MemoryDB;
+	use hashdb::DBValue;
 	use super::triedbmut::TrieDBMut;
 	use super::super::TrieMut;
 

util/src/trie/triedb.rs

@@ -18,16 +18,14 @@ use common::*;
 use hashdb::*;
 use nibbleslice::*;
 use rlp::*;
-use super::node::Node;
+use super::node::{Node, OwnedNode};
-use super::recorder::{Recorder, NoOp};
+use super::lookup::Lookup;
-use super::{Trie, TrieItem, TrieError, TrieIterator};
+use super::{Trie, TrieItem, TrieError, TrieIterator, Query};
 
 /// A `Trie` implementation using a generic `HashDB` backing database.
 ///
-/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys`
+/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object.
-/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get
+/// Use `get` and `contains` to query values associated with keys in the trie.
-/// which items in the backing database do not belong to this trie. If this is the only trie in the
-/// backing database, then `db_items_remaining()` should be empty.
 ///
 /// # Example
 /// ```
@@ -45,7 +43,6 @@ use super::{Trie, TrieItem, TrieError, TrieIterator};
 ///   let t = TrieDB::new(&memdb, &root).unwrap();
 ///   assert!(t.contains(b"foo").unwrap());
 ///   assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar"));
-///   assert!(t.db_items_remaining().unwrap().is_empty());
 /// }
 /// ```
 pub struct TrieDB<'db> {
@@ -76,74 +73,12 @@ impl<'db> TrieDB<'db> {
 		self.db
 	}
 
-	/// Determine all the keys in the backing database that belong to the trie.
-	pub fn keys(&self) -> super::Result<Vec<H256>> {
-		let mut ret: Vec<H256> = Vec::new();
-		ret.push(self.root.clone());
-		self.accumulate_keys(self.root_node(&mut NoOp)?, &mut ret)?;
-		Ok(ret)
-	}
-
-	/// Convert a vector of hashes to a hashmap of hash to occurrences.
-	pub fn to_map(hashes: Vec<H256>) -> HashMap<H256, u32> {
-		let mut r: HashMap<H256, u32> = HashMap::new();
-		for h in hashes {
-			*r.entry(h).or_insert(0) += 1;
-		}
-		r
-	}
-
-	/// Determine occurrences of items in the backing database which are not related to this
-	/// trie.
-	pub fn db_items_remaining(&self) -> super::Result<HashMap<H256, i32>> {
-		let mut ret = self.db.keys();
-		for (k, v) in Self::to_map(self.keys()?) {
-			let keycount = *ret.get(&k).unwrap_or(&0);
-			match keycount <= v as i32 {
-				true => ret.remove(&k),
-				_ => ret.insert(k, keycount - v as i32),
-			};
-		}
-		Ok(ret)
-	}
-
-	/// Recursion helper for `keys`.
-	fn accumulate_keys(&self, node: Node, acc: &mut Vec<H256>) -> super::Result<()> {
-		let mut handle_payload = |payload| {
-			let p = Rlp::new(payload);
-			if p.is_data() && p.size() == 32 {
-				acc.push(p.as_val());
-			}
-
-			self.accumulate_keys(self.get_node(payload, &mut NoOp, 0)?, acc)
-		};
-
-		match node {
-			Node::Extension(_, ref payload) => handle_payload(payload)?,
-			Node::Branch(ref payloads, _) => for payload in payloads { handle_payload(payload)? },
-			_ => {},
-		}
-
-		Ok(())
-	}
-
-	/// Get the root node's RLP.
-	fn root_node<R: Recorder>(&self, r: &mut R) -> super::Result<Node> {
-		self.root_data(r).map(|d| Node::decoded(&d))
-	}
-
 	/// Get the data of the root node.
-	fn root_data<R: Recorder>(&self, r: &mut R) -> super::Result<DBValue> {
+	fn root_data(&self) -> super::Result<DBValue> {
 		self.db.get(self.root).ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root)))
-			.map(|node| { r.record(self.root, &*node, 0); node })
 	}
 
-	/// Get the root node as a `Node`.
+	/// Indentation helper for `format_all`.
-	fn get_node<'a, R: 'a + Recorder>(&'db self, node: &'db [u8], r: &'a mut R, depth: u32) -> super::Result<Node> {
-		self.get_raw_or_lookup(node, r, depth).map(|n| Node::decoded(&n))
-	}
-
-	/// Indentation helper for `formal_all`.
 	fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result {
 		for _ in 0..size {
 			write!(f, "  ")?;
@@ -157,8 +92,8 @@ impl<'db> TrieDB<'db> {
 			Node::Leaf(slice, value) => writeln!(f, "'{:?}: {:?}.", slice, value.pretty())?,
 			Node::Extension(ref slice, ref item) => {
 				write!(f, "'{:?} ", slice)?;
-				if let Ok(node) = self.get_node(&*item, &mut NoOp, 0) {
+				if let Ok(node) = self.get_raw_or_lookup(&*item) {
-					self.fmt_all(node, f, deepness)?;
+					self.fmt_all(Node::decoded(&node), f, deepness)?;
 				}
 			},
 			Node::Branch(ref nodes, ref value) => {
@@ -168,7 +103,8 @@ impl<'db> TrieDB<'db> {
 					writeln!(f, "=: {:?}", v.pretty())?
 				}
 				for i in 0..16 {
-					match self.get_node(&*nodes[i], &mut NoOp, 0) {
+					let node = self.get_raw_or_lookup(&*nodes[i]);
+					match node.as_ref().map(|n| Node::decoded(&*n)) {
 						Ok(Node::Empty) => {},
 						Ok(n) => {
 							self.fmt_indent(f, deepness + 1)?;
@@ -189,64 +125,49 @@ impl<'db> TrieDB<'db> {
 		Ok(())
 	}
 
-	/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
-	fn do_lookup<'key, R: 'key>(&'db self, key: &NibbleSlice<'key>, r: &'key mut R) -> super::Result<Option<DBValue>>
-		where 'db: 'key, R: Recorder
-	{
-		let root_rlp = self.root_data(r)?;
-		self.get_from_node(&root_rlp, key, r, 1)
-	}
-
-	/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
-	/// value exists for the key.
-	///
-	/// Note: Not a public API; use Trie trait functions.
-	fn get_from_node<'key, R: 'key>(
-		&'db self,
-		node: &'db [u8],
-		key: &NibbleSlice<'key>,
-		r: &'key mut R,
-		d: u32
-	) -> super::Result<Option<DBValue>> where 'db: 'key, R: Recorder {
-		match Node::decoded(node) {
-			Node::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == *key => Ok(Some(value.clone())),
-			Node::Extension(ref slice, ref item) => {
-				let slice = &NibbleSlice::from_encoded(slice).0;
-				if key.starts_with(slice) {
-					let data = self.get_raw_or_lookup(&*item, r, d)?;
-					self.get_from_node(&data, &key.mid(slice.len()), r, d + 1)
-				} else {
-					Ok(None)
-				}
-			},
-			Node::Branch(ref nodes, ref value) => match key.is_empty() {
-				true => Ok(value.clone()),
-				false => {
-					let node = self.get_raw_or_lookup(&*nodes[key.at(0) as usize], r, d)?;
-					self.get_from_node(&node, &key.mid(1), r, d + 1)
-				}
-			},
-			_ => Ok(None)
-		}
-	}
-
 	/// Given some node-describing data `node`, return the actual node RLP.
 	/// This could be a simple identity operation in the case that the node is sufficiently small, but
 	/// may require a database lookup.
-	fn get_raw_or_lookup<R: Recorder>(&'db self, node: &'db [u8], rec: &mut R, d: u32) -> super::Result<DBValue> {
+	fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result<DBValue> {
 		// check if its sha3 + len
 		let r = Rlp::new(node);
 		match r.is_data() && r.size() == 32 {
 			true => {
 				let key = r.as_val::<H256>();
 				self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
-					.map(|raw| { rec.record(&key, &raw, d); raw })
 			}
 			false => Ok(DBValue::from_slice(node))
 		}
 	}
 }
 
+impl<'db> Trie for TrieDB<'db> {
+	fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
+		TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>)
+	}
+
+	fn root(&self) -> &H256 { self.root }
+
+	fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result<Option<Q::Item>>
+		where 'a: 'key
+	{
+		Lookup {
+			db: self.db,
+			query: query,
+			hash: self.root.clone(),
+		}.look_up(NibbleSlice::new(key))
+	}
+}
+
+impl<'db> fmt::Debug for TrieDB<'db> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		writeln!(f, "c={:?} [", self.hash_count)?;
+		let root_rlp = self.db.get(self.root).expect("Trie root not found!");
+		self.fmt_all(Node::decoded(&root_rlp), f, 0)?;
+		writeln!(f, "]")
+	}
+}
+
 #[derive(Clone, Eq, PartialEq)]
 enum Status {
 	Entering,
@@ -257,7 +178,7 @@ enum Status {
 
 #[derive(Clone, Eq, PartialEq)]
 struct Crumb {
-	node: Node,
+	node: OwnedNode,
 	status: Status,
 }
 
@@ -265,10 +186,10 @@ impl Crumb {
 	/// Move on to next status in the node's sequence.
 	fn increment(&mut self) {
 		self.status = match (&self.status, &self.node) {
-			(_, &Node::Empty) => Status::Exiting,
+			(_, &OwnedNode::Empty) => Status::Exiting,
 			(&Status::Entering, _) => Status::At,
-			(&Status::At, &Node::Branch(_, _)) => Status::AtChild(0),
+			(&Status::At, &OwnedNode::Branch(_, _)) => Status::AtChild(0),
-			(&Status::AtChild(x), &Node::Branch(_, _)) if x < 15 => Status::AtChild(x + 1),
+			(&Status::AtChild(x), &OwnedNode::Branch(_, _)) if x < 15 => Status::AtChild(x + 1),
 			_ => Status::Exiting,
 		}
 	}
@@ -291,41 +212,40 @@ impl<'a> TrieDBIterator<'a> {
 			key_nibbles: Vec::new(),
 		};
 
-		db.root_data(&mut NoOp).and_then(|root| r.descend(&root))?;
+		db.root_data().and_then(|root| r.descend(&root))?;
 		Ok(r)
 	}
 
-	fn seek_descend<'key> ( &mut self, node: &[u8], key: &NibbleSlice<'key>, d: u32) -> super::Result<()> {
+	fn seek_descend<'key>(&mut self, node_data: DBValue, key: &NibbleSlice<'key>) -> super::Result<()> {
-		match Node::decoded(node) {
+		let node = Node::decoded(&node_data);
+		match node {
 			Node::Leaf(ref slice, _) => {
-				let slice = &NibbleSlice::from_encoded(slice).0;
 				if slice == key {
 					self.trail.push(Crumb {
 						status: Status::At,
-						node: Node::decoded(node),
+						node: node.clone().into(),
 					});
 				} else {
 					self.trail.push(Crumb {
 						status: Status::Exiting,
-						node: Node::decoded(node),
+						node: node.clone().into(),
 					});
 				}
+
 				self.key_nibbles.extend(slice.iter());
 				Ok(())
 			},
 			Node::Extension(ref slice, ref item) => {
-				let slice = &NibbleSlice::from_encoded(slice).0;
 				if key.starts_with(slice) {
-					let mut r = NoOp;
 					self.trail.push(Crumb {
 						status: Status::At,
-						node: Node::decoded(node),
+						node: node.clone().into(),
 					});
 					self.key_nibbles.extend(slice.iter());
-					let data = self.db.get_raw_or_lookup(&*item, &mut r, d)?;
+					let data = self.db.get_raw_or_lookup(&*item)?;
-					self.seek_descend(&data, &key.mid(slice.len()), d + 1)
+					self.seek_descend(data, &key.mid(slice.len()))
 				} else {
-					self.descend(node)?;
+					self.descend(&node_data)?;
 					Ok(())
 				}
 			},
@@ -333,20 +253,19 @@ impl<'a> TrieDBIterator<'a> {
 				true => {
 					self.trail.push(Crumb {
 						status: Status::At,
-						node: Node::decoded(node),
+						node: node.clone().into(),
 					});
 					Ok(())
 				},
 				false => {
-					let mut r = NoOp;
 					let i = key.at(0);
 					self.trail.push(Crumb {
 						status: Status::AtChild(i as usize),
-						node: Node::decoded(node),
+						node: node.clone().into(),
 					});
 					self.key_nibbles.push(i);
-					let child = self.db.get_raw_or_lookup(&*nodes[i as usize], &mut r, d)?;
+					let child = self.db.get_raw_or_lookup(&*nodes[i as usize])?;
-					self.seek_descend(&child, &key.mid(1), d + 1)
+					self.seek_descend(child, &key.mid(1))
 				}
 			},
 			_ => Ok(())
@@ -357,10 +276,12 @@ impl<'a> TrieDBIterator<'a> {
 	fn descend(&mut self, d: &[u8]) -> super::Result<()> {
 		self.trail.push(Crumb {
 			status: Status::Entering,
-			node: self.db.get_node(d, &mut NoOp, 0)?,
+			node: Node::decoded(&self.db.get_raw_or_lookup(d)?).into(),
 		});
-		match self.trail.last().expect("just pushed item; qed").node {
+		match &self.trail.last().expect("just pushed item; qed").node {
-			Node::Leaf(ref n, _) | Node::Extension(ref n, _) => { self.key_nibbles.extend(NibbleSlice::from_encoded(n).0.iter()); },
+			&OwnedNode::Leaf(ref n, _) | &OwnedNode::Extension(ref n, _) => {
+				self.key_nibbles.extend((0..n.len()).map(|i| n.at(i)));
+			},
 			_ => {}
 		}
 
@@ -379,9 +300,8 @@ impl<'a> TrieIterator for TrieDBIterator<'a> {
 	fn seek(&mut self, key: &[u8]) -> super::Result<()> {
 		self.trail.clear();
 		self.key_nibbles.clear();
-		let mut r = NoOp;
+		let root_rlp = self.db.root_data()?;
-		let root_rlp = self.db.root_data(&mut r)?;
+		self.seek_descend(root_rlp, &NibbleSlice::new(key))
-		self.seek_descend(&root_rlp, &NibbleSlice::new(key), 1)
 	}
 }
 
@@ -397,27 +317,27 @@ impl<'a> Iterator for TrieDBIterator<'a> {
 			match (b.status, b.node) {
 				(Status::Exiting, n) => {
 					match n {
-						Node::Leaf(n, _) | Node::Extension(n, _) => {
+						OwnedNode::Leaf(n, _) | OwnedNode::Extension(n, _) => {
 							let l = self.key_nibbles.len();
-							self.key_nibbles.truncate(l - NibbleSlice::from_encoded(&*n).0.len());
+							self.key_nibbles.truncate(l - n.len());
 						},
-						Node::Branch(_, _) => { self.key_nibbles.pop(); },
+						OwnedNode::Branch(_, _) => { self.key_nibbles.pop(); },
 						_ => {}
 					}
 					self.trail.pop();
 					// continue
 				},
-				(Status::At, Node::Leaf(_, v)) | (Status::At, Node::Branch(_, Some(v))) => {
+				(Status::At, OwnedNode::Leaf(_, v)) | (Status::At, OwnedNode::Branch(_, Some(v))) => {
 					return Some(Ok((self.key(), v)));
 				},
-				(Status::At, Node::Extension(_, d)) => {
+				(Status::At, OwnedNode::Extension(_, d)) => {
 					if let Err(e) = self.descend(&*d) {
 						return Some(Err(e));
 					}
 					// continue
 				},
-				(Status::At, Node::Branch(_, _)) => {},
+				(Status::At, OwnedNode::Branch(_, _)) => {},
-				(Status::AtChild(i), Node::Branch(ref children, _)) if children[i].len() > 0 => {
+				(Status::AtChild(i), OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => {
 					match i {
 						0 => self.key_nibbles.push(0),
 						i => *self.key_nibbles.last_mut()
@@ -428,7 +348,7 @@ impl<'a> Iterator for TrieDBIterator<'a> {
 					}
 					// continue
 				},
-				(Status::AtChild(i), Node::Branch(_, _)) => {
+				(Status::AtChild(i), OwnedNode::Branch(_, _)) => {
 					if i == 0 {
 						self.key_nibbles.push(0);
 					}
@@ -440,29 +360,6 @@ impl<'a> Iterator for TrieDBIterator<'a> {
 	}
 }
 
-impl<'db> Trie for TrieDB<'db> {
-	fn iter<'a>(&'a self) -> super::Result<Box<TrieIterator<Item = TrieItem> + 'a>> {
-		TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>)
-	}
-
-	fn root(&self) -> &H256 { self.root }
-
-	fn get_recorded<'a, 'b, R: 'b>(&'a self, key: &'b [u8], rec: &'b mut R) -> super::Result<Option<DBValue>>
-		where 'a: 'b, R: Recorder
-	{
-		self.do_lookup(&NibbleSlice::new(key), rec)
-	}
-}
-
-impl<'db> fmt::Debug for TrieDB<'db> {
-	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-		writeln!(f, "c={:?} [", self.hash_count)?;
-		let root_rlp = self.db.get(self.root).expect("Trie root not found!");
-		self.fmt_all(Node::decoded(&root_rlp), f, 0)?;
-		writeln!(f, "]")
-	}
-}
-
 #[test]
 fn iterator() {
 	use memorydb::*;
@@ -529,3 +426,23 @@ fn iterator_seek() {
 	iter.seek(b"C").unwrap();
 	assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::<Vec<_>>()[..]);
 }
+
+#[test]
+fn get_len() {
+	use memorydb::*;
+	use super::TrieMut;
+	use super::triedbmut::*;
+
+	let mut memdb = MemoryDB::new();
+	let mut root = H256::new();
+	{
+		let mut t = TrieDBMut::new(&mut memdb, &mut root);
+		t.insert(b"A", b"ABC").unwrap();
+		t.insert(b"B", b"ABCBA").unwrap();
+	}
+
+	let t = TrieDB::new(&memdb, &root).unwrap();
+	assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()), Ok(Some(3)));
+	assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()), Ok(Some(5)));
+	assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None));
+}

util/src/trie/triedbmut.rs

@@ -17,6 +17,7 @@
 //! In-memory trie representation.
 
 use super::{TrieError, TrieMut};
+use super::lookup::Lookup;
 use super::node::Node as RlpNode;
 use super::node::NodeKey;
 
@@ -100,22 +101,22 @@ impl Node {
 	fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self {
 		match RlpNode::decoded(rlp) {
 			RlpNode::Empty => Node::Empty,
-			RlpNode::Leaf(k, v) => Node::Leaf(k, v),
+			RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)),
 			RlpNode::Extension(key, cb) => {
-				Node::Extension(key, Self::inline_or_hash(&*cb, db, storage))
+				Node::Extension(key.encoded(false), Self::inline_or_hash(cb, db, storage))
 			}
 			RlpNode::Branch(children_rlp, val) => {
 				let mut children = empty_children();
 
 				for i in 0..16 {
-					let raw = &children_rlp[i];
+					let raw = children_rlp[i];
-					let child_rlp = Rlp::new(&*raw);
+					let child_rlp = Rlp::new(raw);
 					if !child_rlp.is_empty()  {
-						children[i] = Some(Self::inline_or_hash(&*raw, db, storage));
+						children[i] = Some(Self::inline_or_hash(raw, db, storage));
 					}
 				}
 
-				Node::Branch(children, val)
+				Node::Branch(children, val.map(DBValue::from_slice))
 			}
 		}
 	}
@@ -370,7 +371,11 @@ impl<'a> TrieDBMut<'a> {
 		where 'x: 'key
 	{
 		match *handle {
-			NodeHandle::Hash(ref hash) => self.do_db_lookup(hash, partial),
+			NodeHandle::Hash(ref hash) => Lookup {
+				db: &*self.db,
+				query: DBValue::from_slice,
+				hash: hash.clone(),
+			}.look_up(partial),
 			NodeHandle::InMemory(ref handle) => match self.storage[handle] {
 				Node::Empty => Ok(None),
 				Node::Leaf(ref key, ref value) => {
@@ -403,54 +408,6 @@ impl<'a> TrieDBMut<'a> {
 		}
 	}
 
-	/// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists.
-	fn do_db_lookup<'x, 'key>(&'x self, hash: &H256, key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
-		where 'x: 'key
-	{
-		self.db.get(hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(*hash)))
-			.and_then(|node_rlp| self.get_from_db_node(&node_rlp, key))
-	}
-
-	/// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no
-	/// value exists for the key.
-	///
-	/// Note: Not a public API; use Trie trait functions.
-	fn get_from_db_node<'x, 'key>(&'x self, node: &'x [u8], key: NibbleSlice<'key>) -> super::Result<Option<DBValue>>
-		where 'x: 'key
-	{
-		match RlpNode::decoded(node) {
-			RlpNode::Leaf(ref slice, ref value) if NibbleSlice::from_encoded(slice).0 == key => Ok(Some(value.clone())),
-			RlpNode::Extension(ref slice, ref item) => {
-				let slice = &NibbleSlice::from_encoded(slice).0;
-				if key.starts_with(slice) {
-					self.get_from_db_node(&self.get_raw_or_lookup(&*item)?, key.mid(slice.len()))
-				} else {
-					Ok(None)
-				}
-			},
-			RlpNode::Branch(ref nodes, ref value) => match key.is_empty() {
-				true => Ok(value.clone()),
-				false => self.get_from_db_node(&self.get_raw_or_lookup(&*nodes[key.at(0) as usize])?, key.mid(1))
-			},
-			_ => Ok(None),
-		}
-	}
-
-	/// Given some node-describing data `node`, return the actual node RLP.
-	/// This could be a simple identity operation in the case that the node is sufficiently small, but
-	/// may require a database lookup.
-	fn get_raw_or_lookup<'x>(&'x self, node: &'x [u8]) -> super::Result<DBValue> {
-		// check if its sha3 + len
-		let r = Rlp::new(node);
-		match r.is_data() && r.size() == 32 {
-			true => {
-				let key = r.as_val::<H256>();
-				self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key)))
-			}
-			false => Ok(DBValue::from_slice(node))
-		}
-	}
-
 	/// insert a key, value pair into the trie, creating new nodes if necessary.
 	fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option<DBValue>)
 		-> super::Result<(StorageHandle, bool)>