Merge pull request #1452 from ethcore/fatdb

FatDB

util/src/hashdb.rs

@@ -104,6 +104,21 @@ pub trait HashDB: AsHashDB {
 	/// }
 	/// ```
 	fn remove(&mut self, key: &H256);
+
+	/// Insert auxiliary data into hashdb.
+	fn insert_aux(&mut self, _hash: Vec<u8>, _value: Vec<u8>) {
+		unimplemented!();
+	}
+
+	/// Get auxiliary data from hashdb.
+	fn get_aux(&self, _hash: &[u8]) -> Option<Vec<u8>> {
+		unimplemented!();
+	}
+
+	/// Removes auxiliary data from hashdb.
+	fn remove_aux(&mut self, _hash: &[u8]) {
+		unimplemented!();
+	}
 }
 
 /// Upcast trait.

util/src/journaldb/archivedb.rs

@@ -26,6 +26,13 @@ use kvdb::{Database, DBTransaction, DatabaseConfig};
 #[cfg(test)]
 use std::env;
 
+/// Suffix appended to auxiliary keys to distinguish them from normal keys.
+/// Would be nich to use rocksdb columns for this eventually.
+const AUX_FLAG: u8 = 255;
+
+/// Database version.
+const DB_VERSION : u32 = 0x103;
+
 /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay
 /// and latent-removal semantics.
 ///
@@ -39,8 +46,6 @@ pub struct ArchiveDB {
 	latest_era: Option<u64>,
 }
 
-const DB_VERSION : u32 = 0x103;
-
 impl ArchiveDB {
 	/// Create a new instance from file
 	pub fn new(path: &str, config: DatabaseConfig) -> ArchiveDB {
@@ -115,12 +120,35 @@ impl HashDB for ArchiveDB {
 	fn insert(&mut self, value: &[u8]) -> H256 {
 		self.overlay.insert(value)
 	}
+
 	fn emplace(&mut self, key: H256, value: Bytes) {
 		self.overlay.emplace(key, value);
 	}
+
 	fn remove(&mut self, key: &H256) {
 		self.overlay.remove(key);
 	}
+
+	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
+		self.overlay.insert_aux(hash, value);
+	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		if let Some(res) = self.overlay.get_aux(hash) {
+			return Some(res)
+		}
+
+		let mut db_hash = hash.to_vec();
+		db_hash.push(AUX_FLAG);
+
+		self.backing.get(&db_hash)
+			.expect("Low-level database error. Some issue with your hard disk?")
+			.map(|v| v.to_vec())
+	}
+
+	fn remove_aux(&mut self, hash: &[u8]) {
+		self.overlay.remove_aux(hash);
+	}
 }
 
 impl JournalDB for ArchiveDB {
@@ -144,6 +172,7 @@ impl JournalDB for ArchiveDB {
 		let batch = DBTransaction::new();
 		let mut inserts = 0usize;
 		let mut deletes = 0usize;
+
 		for i in self.overlay.drain().into_iter() {
 			let (key, (value, rc)) = i;
 			if rc > 0 {
@@ -156,6 +185,12 @@ impl JournalDB for ArchiveDB {
 				deletes += 1;
 			}
 		}
+
+		for (mut key, value) in self.overlay.drain_aux().into_iter() {
+			key.push(AUX_FLAG);
+			batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?");
+		}
+
 		if self.latest_era.map_or(true, |e| now > e) {
 			try!(batch.put(&LATEST_ERA_KEY, &encode(&now)));
 			self.latest_era = Some(now);

util/src/memorydb.rs

@@ -74,6 +74,7 @@ use std::default::Default;
 pub struct MemoryDB {
 	data: HashMap<H256, (Bytes, i32)>,
 	static_null_rlp: (Bytes, i32),
+	aux: HashMap<Bytes, Bytes>,
 }
 
 impl Default for MemoryDB {
@@ -88,6 +89,7 @@ impl MemoryDB {
 		MemoryDB {
 			data: HashMap::new(),
  			static_null_rlp: (vec![0x80u8; 1], 1),
+			aux: HashMap::new(),
 		}
 	}
 
@@ -134,9 +136,12 @@ impl MemoryDB {
 
 	/// Return the internal map of hashes to data, clearing the current state.
 	pub fn drain(&mut self) -> HashMap<H256, (Bytes, i32)> {
-		let mut data = HashMap::new();
+		mem::replace(&mut self.data, HashMap::new())
-		mem::swap(&mut self.data, &mut data);
+	}
-		data
+
+	/// Return the internal map of auxiliary data, clearing the current state.
+	pub fn drain_aux(&mut self) -> HashMap<Bytes, Bytes> {
+		mem::replace(&mut self.aux, HashMap::new())
 	}
 
 	/// Denote than an existing value has the given key. Used when a key gets removed without
@@ -233,6 +238,18 @@ impl HashDB for MemoryDB {
 			self.data.insert(key.clone(), (Bytes::new(), -1));
 		}
 	}
+
+	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
+		self.aux.insert(hash, value);
+	}
+
+	fn get_aux(&self, hash: &[u8]) -> Option<Vec<u8>> {
+		self.aux.get(hash).cloned()
+	}
+
+	fn remove_aux(&mut self, hash: &[u8]) {
+		self.aux.remove(hash);
+	}
 }
 
 #[test]

util/src/trie/fatdb.rs

@@ -0,0 +1,112 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use hash::H256;
+use sha3::Hashable;
+use hashdb::HashDB;
+use super::{TrieDB, Trie, TrieDBIterator, TrieError};
+
+/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
+/// Additionaly it stores inserted hash-key mappings for later retrieval.
+///
+/// Use it as a `Trie` or `TrieMut` trait object.
+pub struct FatDB<'db> {
+	raw: TrieDB<'db>,
+}
+
+impl<'db> FatDB<'db> {
+	/// Create a new trie with the backing database `db` and empty `root`
+	/// Initialise to the state entailed by the genesis block.
+	/// This guarantees the trie is built correctly.
+	pub fn new(db: &'db HashDB, root: &'db H256) -> Result<Self, TrieError> {
+		let fatdb = FatDB {
+			raw: try!(TrieDB::new(db, root))
+		};
+
+		Ok(fatdb)
+	}
+
+	/// Get the backing database.
+	pub fn db(&self) -> &HashDB {
+		self.raw.db()
+	}
+
+	/// Iterator over all key / vlaues in the trie.
+	pub fn iter(&self) -> FatDBIterator {
+		FatDBIterator::new(&self.raw)
+	}
+}
+
+impl<'db> Trie for FatDB<'db> {
+	fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
+		Box::new(FatDB::iter(self))
+	}
+
+	fn root(&self) -> &H256 {
+		self.raw.root()
+	}
+
+	fn contains(&self, key: &[u8]) -> bool {
+		self.raw.contains(&key.sha3())
+	}
+
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
+		self.raw.get(&key.sha3())
+	}
+}
+
+/// Itarator over inserted pairs of key values.
+pub struct FatDBIterator<'db> {
+	trie_iterator: TrieDBIterator<'db>,
+	trie: &'db TrieDB<'db>,
+}
+
+impl<'db> FatDBIterator<'db> {
+	/// Creates new iterator.
+	pub fn new(trie: &'db TrieDB) -> Self {
+		FatDBIterator {
+			trie_iterator: TrieDBIterator::new(trie),
+			trie: trie,
+		}
+	}
+}
+
+impl<'db> Iterator for FatDBIterator<'db> {
+	type Item = (Vec<u8>, &'db [u8]);
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.trie_iterator.next()
+			.map(|(hash, value)| {
+				(self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value)
+			})
+	}
+}
+
+#[test]
+fn fatdb_to_trie() {
+	use memorydb::MemoryDB;
+	use trie::{FatDBMut, TrieMut};
+
+	let mut memdb = MemoryDB::new();
+	let mut root = H256::default();
+	{
+		let mut t = FatDBMut::new(&mut memdb, &mut root);
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+	}
+	let t = FatDB::new(&memdb, &root).unwrap();
+	assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), &[0x01u8, 0x23]);
+	assert_eq!(t.iter().collect::<Vec<_>>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]);
+}

util/src/trie/fatdbmut.rs

@@ -0,0 +1,94 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use hash::H256;
+use sha3::Hashable;
+use hashdb::HashDB;
+use super::{TrieDBMut, Trie, TrieMut, TrieError};
+
+/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database.
+/// Additionaly it stores inserted hash-key mappings for later retrieval.
+///
+/// Use it as a `Trie` or `TrieMut` trait object.
+pub struct FatDBMut<'db> {
+	raw: TrieDBMut<'db>,
+}
+
+impl<'db> FatDBMut<'db> {
+	/// Create a new trie with the backing database `db` and empty `root`
+	/// Initialise to the state entailed by the genesis block.
+	/// This guarantees the trie is built correctly.
+	pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self {
+		FatDBMut { raw: TrieDBMut::new(db, root) }
+	}
+
+	/// Create a new trie with the backing database `db` and `root`.
+	///
+	/// Returns an error if root does not exist.
+	pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result<Self, TrieError> {
+		Ok(FatDBMut { raw: try!(TrieDBMut::from_existing(db, root)) })
+	}
+
+	/// Get the backing database.
+	pub fn db(&self) -> &HashDB {
+		self.raw.db()
+	}
+
+	/// Get the backing database.
+	pub fn db_mut(&mut self) -> &mut HashDB {
+		self.raw.db_mut()
+	}
+}
+
+impl<'db> TrieMut for FatDBMut<'db> {
+	fn root(&self) -> &H256 {
+		self.raw.root()
+	}
+
+	fn contains(&self, key: &[u8]) -> bool {
+		self.raw.contains(&key.sha3())
+	}
+
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
+		self.raw.get(&key.sha3())
+	}
+
+	fn insert(&mut self, key: &[u8], value: &[u8]) {
+		let hash = key.sha3();
+		self.raw.insert(&hash, value);
+		let db = self.raw.db_mut();
+		db.insert_aux(hash.to_vec(), key.to_vec());
+	}
+
+	fn remove(&mut self, key: &[u8]) {
+		self.raw.remove(&key.sha3());
+	}
+}
+
+#[test]
+fn fatdb_to_trie() {
+	use memorydb::MemoryDB;
+	use super::TrieDB;
+
+	let mut memdb = MemoryDB::new();
+	let mut root = H256::default();
+	{
+		let mut t = FatDBMut::new(&mut memdb, &mut root);
+		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);
+	}
+	let t = TrieDB::new(&memdb, &root).unwrap();
+	assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]);
+}

util/src/trie/mod.rs

@@ -17,6 +17,8 @@
 //! Trie interface and implementation.
 
 use std::fmt;
+use hash::H256;
+use hashdb::HashDB;
 
 /// Export the trietraits module.
 pub mod trietraits;
@@ -35,12 +37,18 @@ pub mod sectriedb;
 /// Export the sectriedbmut module.
 pub mod sectriedbmut;
 
+mod fatdb;
+
+mod fatdbmut;
+
 pub use self::trietraits::{Trie, TrieMut};
 pub use self::standardmap::{Alphabet, StandardMap, ValueMode};
 pub use self::triedbmut::TrieDBMut;
-pub use self::triedb::TrieDB;
+pub use self::triedb::{TrieDB, TrieDBIterator};
 pub use self::sectriedbmut::SecTrieDBMut;
 pub use self::sectriedb::SecTrieDB;
+pub use self::fatdb::{FatDB, FatDBIterator};
+pub use self::fatdbmut::FatDBMut;
 
 /// Trie Errors
 #[derive(Debug)]
@@ -53,4 +61,63 @@ impl fmt::Display for TrieError {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		write!(f, "Trie Error: Invalid state root.")
 	}
-}
+}
+
+/// Trie types
+#[derive(Debug, Clone)]
+pub enum TrieSpec {
+	/// Generic trie.
+	Generic,
+	/// Secure trie.
+	Secure,
+	///	Secure trie with fat database.
+	Fat,
+}
+
+impl Default for TrieSpec {
+	fn default() -> TrieSpec {
+		TrieSpec::Secure
+	}
+}
+
+/// Trie factory.
+#[derive(Default, Clone)]
+pub struct TrieFactory {
+	spec: TrieSpec,
+}
+
+impl TrieFactory {
+	/// Creates new factory.
+	pub fn new(spec: TrieSpec) -> Self {
+		TrieFactory {
+			spec: spec,
+		}
+	}
+
+	/// Create new immutable instance of Trie.
+	pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result<Box<Trie + 'db>, TrieError> {
+		match self.spec {
+			TrieSpec::Generic => Ok(Box::new(try!(TrieDB::new(db, root)))),
+			TrieSpec::Secure => Ok(Box::new(try!(SecTrieDB::new(db, root)))),
+			TrieSpec::Fat => Ok(Box::new(try!(FatDB::new(db, root)))),
+		}
+	}
+
+	/// Create new mutable instance of Trie.
+	pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box<TrieMut + 'db> {
+		match self.spec {
+			TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)),
+			TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)),
+			TrieSpec::Fat => Box::new(FatDBMut::new(db, root)),
+		}
+	}
+
+	/// Create new mutable instance of trie and check for errors.
+	pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result<Box<TrieMut + 'db>, TrieError> {
+		match self.spec {
+			TrieSpec::Generic => Ok(Box::new(try!(TrieDBMut::from_existing(db, root)))),
+			TrieSpec::Secure => Ok(Box::new(try!(SecTrieDBMut::from_existing(db, root)))),
+			TrieSpec::Fat => Ok(Box::new(try!(FatDBMut::from_existing(db, root)))),
+		}
+	}
+}

util/src/trie/sectriedb.rs

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use hash::*;
+use hash::H256;
-use sha3::*;
+use sha3::Hashable;
 use hashdb::HashDB;
 use super::triedb::TrieDB;
 use super::trietraits::Trie;
@@ -50,6 +50,10 @@ impl<'db> SecTrieDB<'db> {
 }
 
 impl<'db> Trie for SecTrieDB<'db> {
+	fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
+		Box::new(TrieDB::iter(&self.raw))
+	}
+
 	fn root(&self) -> &H256 { self.raw.root() }
 
 	fn contains(&self, key: &[u8]) -> bool {
@@ -68,7 +72,7 @@ fn trie_to_sectrie() {
 	use super::trietraits::TrieMut;
 
 	let mut memdb = MemoryDB::new();
-	let mut root = H256::new();
+	let mut root = H256::default();
 	{
 		let mut t = TrieDBMut::new(&mut memdb, &mut root);
 		t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]);

util/src/trie/sectriedbmut.rs

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use hash::*;
+use hash::H256;
-use sha3::*;
+use sha3::Hashable;
 use hashdb::HashDB;
 use super::triedbmut::TrieDBMut;
 use super::trietraits::{Trie, TrieMut};
@@ -44,13 +44,13 @@ impl<'db> SecTrieDBMut<'db> {
 	}
 
 	/// Get the backing database.
-	pub fn db(&'db self) -> &'db HashDB { self.raw.db() }
+	pub fn db(&self) -> &HashDB { self.raw.db() }
 
 	/// Get the backing database.
-	pub fn db_mut(&'db mut self) -> &'db mut HashDB { self.raw.db_mut() }
+	pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() }
 }
 
-impl<'db> Trie for SecTrieDBMut<'db> {
+impl<'db> TrieMut for SecTrieDBMut<'db> {
 	fn root(&self) -> &H256 { self.raw.root() }
 
 	fn contains(&self, key: &[u8]) -> bool {
@@ -60,9 +60,7 @@ impl<'db> Trie for SecTrieDBMut<'db> {
 	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
 		self.raw.get(&key.sha3())
 	}
-}
 
-impl<'db> TrieMut for SecTrieDBMut<'db> {
 	fn insert(&mut self, key: &[u8], value: &[u8]) {
 		self.raw.insert(&key.sha3(), value);
 	}
@@ -78,7 +76,7 @@ fn sectrie_to_trie() {
 	use super::triedb::*;
 
 	let mut memdb = MemoryDB::new();
-	let mut root = H256::new();
+	let mut root = H256::default();
 	{
 		let mut t = SecTrieDBMut::new(&mut memdb, &mut root);
 		t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]);

util/src/trie/triedb.rs

@@ -18,7 +18,7 @@ use common::*;
 use hashdb::*;
 use nibbleslice::*;
 use rlp::*;
-use super::trietraits::Trie;
+use super::trietraits::{Trie};
 use super::node::Node;
 use super::TrieError;
 
@@ -257,7 +257,7 @@ pub struct TrieDBIterator<'a> {
 
 impl<'a> TrieDBIterator<'a> {
 	/// Create a new iterator.
-	fn new(db: &'a TrieDB) -> TrieDBIterator<'a> {
+	pub fn new(db: &'a TrieDB) -> TrieDBIterator<'a> {
 		let mut r = TrieDBIterator {
 			db: db,
 			trail: vec![],
@@ -331,10 +331,16 @@ impl<'a> Iterator for TrieDBIterator<'a> {
 
 impl<'db> TrieDB<'db> {
 	/// Get all keys/values stored in the trie.
-	pub fn iter(&self) -> TrieDBIterator { TrieDBIterator::new(self) }
+	pub fn iter(&self) -> TrieDBIterator {
+		TrieDBIterator::new(self)
+	}
 }
 
 impl<'db> Trie for TrieDB<'db> {
+	fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a> {
+		Box::new(TrieDB::iter(self))
+	}
+
 	fn root(&self) -> &H256 { &self.root }
 
 	fn contains(&self, key: &[u8]) -> bool {

util/src/trie/triedbmut.rs

@@ -99,12 +99,12 @@ impl<'db> TrieDBMut<'db> {
 	}
 
 	/// Get the backing database.
-	pub fn db(&'db self) -> &'db HashDB {
+	pub fn db(&self) -> &HashDB {
 		self.db
 	}
 
 	/// Get the backing database.
-	pub fn db_mut(&'db mut self) -> &'db mut HashDB {
+	pub fn db_mut(&mut self) -> &mut HashDB {
 		self.db
 	}
 
@@ -642,7 +642,7 @@ impl<'db> TrieDBMut<'db> {
 	}
 }
 
-impl<'db> Trie for TrieDBMut<'db> {
+impl<'db> TrieMut for TrieDBMut<'db> {
 	fn root(&self) -> &H256 { &self.root }
 
 	fn contains(&self, key: &[u8]) -> bool {
@@ -652,9 +652,7 @@ impl<'db> Trie for TrieDBMut<'db> {
 	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key {
 		self.do_lookup(&NibbleSlice::new(key))
 	}
-}
 
-impl<'db> TrieMut for TrieDBMut<'db> {
 	fn insert(&mut self, key: &[u8], value: &[u8]) {
 		match value.is_empty() {
 			false => self.insert_ns(&NibbleSlice::new(key), value),

util/src/trie/trietraits.rs

@@ -30,10 +30,25 @@ pub trait Trie {
 
 	/// What is the value of the given key in this trie?
 	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key;
+
+	/// Returns an iterator over elements of trie.
+	fn iter<'a>(&'a self) -> Box<Iterator<Item = (Vec<u8>, &[u8])> + 'a>;
 }
 
 /// A key-value datastore implemented as a database-backed modified Merkle tree.
-pub trait TrieMut: Trie {
+pub trait TrieMut {
+	/// Return the root of the trie.
+	fn root(&self) -> &H256;
+
+	/// Is the trie empty?
+	fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP }
+
+	/// Does the trie contain a given key?
+	fn contains(&self, key: &[u8]) -> bool;
+
+	/// What is the value of the given key in this trie?
+	fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key;
+
 	/// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing
 	/// `key` from the trie.
 	fn insert(&mut self, key: &[u8], value: &[u8]);
@@ -42,4 +57,3 @@ pub trait TrieMut: Trie {
 	/// value.
 	fn remove(&mut self, key: &[u8]);
 }
-