fatdb and fatdb iterator module

util/src/hashdb.rs

@@ -104,6 +104,16 @@ pub trait HashDB: AsHashDB {
 	/// }
 	/// ```
 	fn remove(&mut self, key: &H256);
+
+	/// Insert auxiliary data into hashdb.
+	fn insert_aux(&mut self, _hash: Vec<u8>, _value: Vec<u8>) {
+		unimplemented!();
+	}
+
+	/// Get auxiliary data from hashdb.
+	fn get_aux(&self, _hash: &[u8]) -> Option<Vec<u8>> {
+		unimplemented!();
+	}
 }
 
 /// Upcast trait.

util/src/journaldb/archivedb.rs

@@ -26,6 +26,8 @@ use kvdb::{Database, DBTransaction, DatabaseConfig};
 #[cfg(test)]
 use std::env;
 
+const AUX_FLAG: u8 = 255;
+
 /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
 /// and latent-removal semantics.
 ///
@@ -126,6 +128,23 @@ impl HashDB for ArchiveDB {
 	fn remove(&mut self, key: &H256) {
 		self.overlay.remove(key);
 	}
+
+	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
+		self.overlay.insert_aux(hash, value);
+	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		if let Some(res) = self.overlay.get_aux(hash) {
+			return Some(res)
+		}
+
+		let mut db_hash = hash.to_vec();
+		db_hash.push(AUX_FLAG);
+
+		self.backing.get(&db_hash)
+			.expect("Low-level database error. Some issue with your hard disk?")
+			.map(|v| v.to_vec())
+	}
 }
 
 impl JournalDB for ArchiveDB {
@@ -149,6 +168,7 @@ impl JournalDB for ArchiveDB {
 		let batch = DBTransaction::new();
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
+
 		for i in self.overlay.drain().into_iter() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
@@ -161,6 +181,12 @@ impl JournalDB for ArchiveDB {
 				deletes += 1;
 			}
 		}
+
+		for (mut key, value) in self.overlay.drain_aux().into_iter() {
+			key.push(AUX_FLAG);
+			batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
+		}
+
 		if self.latest_era.map_or(true, |e| now > e) {
 			try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
 			self.latest_era = Some(now);

util/src/memorydb.rs

@@ -74,6 +74,7 @@ use std::default::Default;
 pub struct MemoryDB {
 	data: HashMap<H256, (Bytes, i32)>,
 	static_null_rlp: (Bytes, i32),
+	aux: HashMap<Bytes, Bytes>,
 }
 
 impl Default for MemoryDB {
@@ -88,6 +89,7 @@ impl MemoryDB {
 		MemoryDB {
 			data: HashMap::new(),
  			static_null_rlp: (vec![0x80u8; 1], 1),
+			aux: HashMap::new(),
 		}
 	}
 
@@ -139,6 +141,13 @@ impl MemoryDB {
 		data
 	}
 
+	/// Return the internal map of auxiliary data, clearing the current state.
+	pub fn drain_aux(&mut self) -> HashMap<Bytes, Bytes> {
+		let mut aux = HashMap::new();
+		mem::swap(&mut self.aux, &mut aux);
+		aux
+	}
+
 	/// Denote than an existing value has the given key. Used when a key gets removed without
 	/// a prior insert and thus has a negative reference with no value.
 	///
@@ -233,6 +242,14 @@ impl HashDB for MemoryDB {
 			self.data.insert(key.clone(), (Bytes::new(), -1));
 		}
 	}
+
+	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
+		self.aux.insert(hash, value);
+	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		self.aux.get(hash).cloned()
+	}
 }
 
 #[test]

util/src/trie/fatdb.rs

@@ -0,0 +1,122 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use hash::H256;
+use sha3::Hashable;
+use hashdb::HashDB;
+use super::{TrieDBMut, Trie, TrieDB, TrieMut, TrieDBIterator, TrieError};
+
+/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
+///
+/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object.
+pub struct FatDB<'db> {
+	raw: TrieDBMut<'db>,
+}
+
+impl<'db> FatDB<'db> {
+	/// Create a new trie with the backing database `db` and empty `root`
+	/// Initialise to the state entailed by the genesis block.
+	/// This guarantees the trie is built correctly.
+	pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
+		FatDB { raw: TrieDBMut::new(db, root) }
+	}
+
+	/// Create a new trie with the backing database `db` and `root`.
+	///
+	/// Returns an error if root does not exist.
+	pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result<Self, TrieError> {
+		Ok(FatDB { raw: try!(TrieDBMut::from_existing(db, root)) })
+	}
+
+	/// Get the backing database.
+	pub fn db(&self) -> &HashDB {
+		self.raw.db()
+	}
+
+	/// Get the backing database.
+	pub fn db_mut(&mut self) -> &mut HashDB {
+		self.raw.db_mut()
+	}
+}
+
+impl<'db> Trie for FatDB<'db> {
+	fn root(&self) -> &H256 {
+		self.raw.root()
+	}
+
+	fn contains(&self, key: &[u8]) -> bool {
+		self.raw.contains(&key.sha3())
+	}
+
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
+		self.raw.get(&key.sha3())
+	}
+}
+
+impl<'db> TrieMut for FatDB<'db> {
+	fn insert(&mut self, key: &[u8], value: &[u8]) {
+		let hash = key.sha3();
+		self.raw.insert(&hash, value);
+		let db = self.raw.db_mut();
+		db.insert_aux(hash.to_vec(), key.to_vec());
+	}
+
+	fn remove(&mut self, key: &[u8]) {
+		self.raw.remove(&key.sha3());
+	}
+}
+
+/// Itarator over inserted pairs of key values.
+pub struct FatDBIterator<'db> {
+	trie_iterator: TrieDBIterator<'db>,
+	trie: &'db TrieDB<'db>,
+}
+
+impl<'db> FatDBIterator<'db> {
+	pub fn new(trie: &'db TrieDB) -> Self {
+		FatDBIterator {
+			trie_iterator: TrieDBIterator::new(trie),
+			trie: trie,
+		}
+	}
+}
+
+impl<'db> Iterator for FatDBIterator<'db> {
+	type Item = (Vec<u8>, &'db [u8]);
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.trie_iterator.next()
+			.map(|(hash, value)| {
+				(self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value)
+			})
+	}
+}
+
+#[test]
+fn fatdb_to_trie() {
+	use memorydb::MemoryDB;
+	use super::TrieDB;
+
+	let mut memdb = MemoryDB::new();
+	let mut root = H256::default();
+	{
+		let mut t = FatDB::new(&mut memdb, &mut root);
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+	}
+	let t = TrieDB::new(&memdb, &root).unwrap();
+	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]);
+	assert_eq!(FatDBIterator::new(&t).collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]);
+}

util/src/trie/mod.rs

@@ -35,12 +35,15 @@ pub mod sectriedb;
 /// Export the sectriedbmut module.
 pub mod sectriedbmut;
 
+mod fatdb;
+
 pub use self::trietraits::{Trie, TrieMut};
 pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
 pub use self::triedbmut::TrieDBMut;
-pub use self::triedb::TrieDB;
+pub use self::triedb::{TrieDB, TrieDBIterator};
 pub use self::sectriedbmut::SecTrieDBMut;
 pub use self::sectriedb::SecTrieDB;
+pub use self::fatdb::{FatDB, FatDBIterator};
 
 /// Trie Errors
 #[derive(Debug)]

util/src/trie/sectriedb.rs

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use hash::*;
+use hash::H256;
-use sha3::*;
+use sha3::Hashable;
 use hashdb::HashDB;
 use super::triedb::TrieDB;
 use super::trietraits::Trie;
@@ -68,7 +68,7 @@ fn trie_to_sectrie() {
 	use super::trietraits::TrieMut;
 
 	let mut memdb = MemoryDB::new();
-	let mut root = H256::new();
+	let mut root = H256::default();
 	{
 		let mut t = TrieDBMut::new(&mut memdb, &mut root);
 		t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]);

util/src/trie/sectriedbmut.rs

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use hash::*;
+use hash::H256;
-use sha3::*;
+use sha3::Hashable;
 use hashdb::HashDB;
 use super::triedbmut::TrieDBMut;
 use super::trietraits::{Trie, TrieMut};
@@ -44,10 +44,10 @@ impl<'db> SecTrieDBMut<'db> {
 	}
 
 	/// Get the backing database.
-	pub fn db(&'db self) -> &'db HashDB { self.raw.db() }
+	pub fn db(&self) -> &HashDB { self.raw.db() }
 
 	/// Get the backing database.
-	pub fn db_mut(&'db mut self) -> &'db mut HashDB { self.raw.db_mut() }
+	pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() }
 }
 
 impl<'db> Trie for SecTrieDBMut<'db> {
@@ -78,7 +78,7 @@ fn sectrie_to_trie() {
 	use super::triedb::*;
 
 	let mut memdb = MemoryDB::new();
-	let mut root = H256::new();
+	let mut root = H256::default();
 	{
 		let mut t = SecTrieDBMut::new(&mut memdb, &mut root);
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);

util/src/trie/triedb.rs

@@ -257,7 +257,7 @@ pub struct TrieDBIterator<'a> {
 
 impl<'a> TrieDBIterator<'a> {
 	/// Create a new iterator.
-	fn new(db: &'a TrieDB) -> TrieDBIterator<'a> {
+	pub fn new(db: &'a TrieDB) -> TrieDBIterator<'a> {
 		let mut r = TrieDBIterator {
 			db: db,
 			trail: vec![],

util/src/trie/triedbmut.rs

@@ -99,12 +99,12 @@ impl<'db> TrieDBMut<'db> {
 	}
 
 	/// Get the backing database.
-	pub fn db(&'db self) -> &'db HashDB {
+	pub fn db(&self) -> &HashDB {
 		self.db
 	}
 
 	/// Get the backing database.
-	pub fn db_mut(&'db mut self) -> &'db mut HashDB {
+	pub fn db_mut(&mut self) -> &mut HashDB {
 		self.db
 	}