Merge pull request #3636 from ethcore/fatdb

Return of the Fat DB

ethcore/src/account_db.rs

@@ -121,10 +121,6 @@ impl<'db> HashDB for AccountDB<'db>{
 	fn remove(&mut self, _key: &H256) {
 		unimplemented!()
 	}
-
-	fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
-		self.db.get_aux(hash)
-	}
 }
 
 /// DB backend wrapper for Account trie
@@ -197,18 +193,6 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		let key = combine_key(&self.address_hash, key);
 		self.db.remove(&key)
 	}
-
-	fn insert_aux(&mut self, hash: Vec<u8>, value: Vec<u8>) {
-		self.db.insert_aux(hash, value);
-	}
-
-	fn get_aux(&self, hash: &[u8]) -> Option<DBValue> {
-		self.db.get_aux(hash)
-	}
-
-	fn remove_aux(&mut self, hash: &[u8]) {
-		self.db.remove_aux(hash);
-	}
 }
 
 struct Wrapping<'db>(&'db HashDB);

ethcore/src/block.rs

@@ -594,9 +594,9 @@ mod tests {
 	use factory::Factories;
 	use state_db::StateDB;
 	use views::BlockView;
-	use util::Address;
+	use util::{Address, TrieFactory};
 	use util::hash::FixedHash;
-
+	use util::trie::TrieSpec;
 	use std::sync::Arc;
 
 	/// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header
@@ -637,7 +637,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(&*spec.engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();
@@ -653,7 +653,7 @@ mod tests {
 
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap()
 			.close_and_lock().seal(engine, vec![]).unwrap();
@@ -662,7 +662,7 @@ mod tests {
 
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap();
 
 		assert_eq!(e.rlp_bytes(), orig_bytes);
@@ -681,7 +681,7 @@ mod tests {
 
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let mut open_block = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle1_header = Header::new();
@@ -697,7 +697,7 @@ mod tests {
 
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap();
 
 		let bytes = e.rlp_bytes();

ethcore/src/client/client.rs

@@ -22,7 +22,7 @@ use std::time::{Instant};
 use time::precise_time_ns;
 
 // util
-use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock};
+use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock, Hashable};
 use util::{journaldb, TrieFactory, Trie};
 use util::trie::TrieSpec;
 use util::{U256, H256, Address, H2048, Uint, FixedHash};
@@ -172,9 +172,10 @@ impl Client {
 			false => TrieSpec::Secure,
 		};
 
+		let trie_factory = TrieFactory::new(trie_spec);
 		let journal_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
 		let mut state_db = StateDB::new(journal_db, config.state_cache_size);
-		if state_db.journal_db().is_empty() && try!(spec.ensure_db_good(&mut state_db)) {
+		if state_db.journal_db().is_empty() && try!(spec.ensure_db_good(&mut state_db, &trie_factory)) {
 			let mut batch = DBTransaction::new(&db);
 			try!(state_db.journal_under(&mut batch, 0, &spec.genesis_header().hash()));
 			try!(db.write(batch).map_err(ClientError::Database));
@@ -216,7 +217,7 @@ impl Client {
 
 		let factories = Factories {
 			vm: EvmFactory::new(config.vm_type.clone(), config.jump_table_size),
-			trie: TrieFactory::new(trie_spec),
+			trie: trie_factory,
 			accountdb: Default::default(),
 		};
 
@@ -952,6 +953,10 @@ impl BlockChainClient for Client {
 		self.state_at(id).map(|s| s.nonce(address))
 	}
 
+	fn storage_root(&self, address: &Address, id: BlockID) -> Option<H256> {
+		self.state_at(id).and_then(|s| s.storage_root(address))
+	}
+
 	fn block_hash(&self, id: BlockID) -> Option<H256> {
 		let chain = self.chain.read();
 		Self::block_hash(&chain, id)
@@ -969,7 +974,7 @@ impl BlockChainClient for Client {
 		self.state_at(id).map(|s| s.storage_at(address, position))
 	}
 
-	fn list_accounts(&self, id: BlockID) -> Option<Vec<Address>> {
+	fn list_accounts(&self, id: BlockID, after: Option<&Address>, count: u64) -> Option<Vec<Address>> {
 		if !self.factories.trie.is_fat() {
 			trace!(target: "fatdb", "list_accounts: Not a fat DB");
 			return None;
@@ -989,18 +994,68 @@ impl BlockChainClient for Client {
 			}
 		};
 
-		let iter = match trie.iter() {
+		let mut iter = match trie.iter() {
 			Ok(iter) => iter,
 			_ => return None,
 		};
 
+		if let Some(after) = after {
+			if let Err(e) = iter.seek(after) {
+				trace!(target: "fatdb", "list_accounts: Couldn't seek the DB: {:?}", e);
+			}
+		}
+
 		let accounts = iter.filter_map(|item| {
 			item.ok().map(|(addr, _)| Address::from_slice(&addr))
-		}).collect();
+		}).take(count as usize).collect();
 
 		Some(accounts)
 	}
 
+	fn list_storage(&self, id: BlockID, account: &Address, after: Option<&H256>, count: u64) -> Option<Vec<H256>> {
+		if !self.factories.trie.is_fat() {
+			trace!(target: "fatdb", "list_stroage: Not a fat DB");
+			return None;
+		}
+
+		let state = match self.state_at(id) {
+			Some(state) => state,
+			_ => return None,
+		};
+
+		let root = match state.storage_root(account) {
+			Some(root) => root,
+			_ => return None,
+		};
+
+		let (_, db) = state.drop();
+		let account_db = self.factories.accountdb.readonly(db.as_hashdb(), account.sha3());
+		let trie = match self.factories.trie.readonly(account_db.as_hashdb(), &root) {
+			Ok(trie) => trie,
+			_ => {
+				trace!(target: "fatdb", "list_storage: Couldn't open the DB");
+				return None;
+			}
+		};
+
+		let mut iter = match trie.iter() {
+			Ok(iter) => iter,
+			_ => return None,
+		};
+
+		if let Some(after) = after {
+			if let Err(e) = iter.seek(after) {
+				trace!(target: "fatdb", "list_accounts: Couldn't seek the DB: {:?}", e);
+			}
+		}
+
+		let keys = iter.filter_map(|item| {
+			item.ok().map(|(key, _)| H256::from_slice(&key))
+		}).take(count as usize).collect();
+
+		Some(keys)
+	}
+
 	fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction> {
 		self.transaction_address(id).and_then(|address| self.chain.read().transaction(&address))
 	}

ethcore/src/client/test_client.rs

@@ -333,7 +333,7 @@ impl MiningBlockChainClient for TestBlockChainClient {
 		let genesis_header = self.spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		self.spec.ensure_db_good(&mut db).unwrap();
+		self.spec.ensure_db_good(&mut db, &TrieFactory::default()).unwrap();
 
 		let last_hashes = vec![genesis_header.hash()];
 		let mut open_block = OpenBlock::new(
@@ -385,6 +385,10 @@ impl BlockChainClient for TestBlockChainClient {
 		}
 	}
 
+	fn storage_root(&self, _address: &Address, _id: BlockID) -> Option<H256> {
+		None
+	}
+
 	fn latest_nonce(&self, address: &Address) -> U256 {
 		self.nonce(address, BlockID::Latest).unwrap()
 	}
@@ -416,10 +420,13 @@ impl BlockChainClient for TestBlockChainClient {
 		}
 	}
 
-	fn list_accounts(&self, _id: BlockID) -> Option<Vec<Address>> {
+	fn list_accounts(&self, _id: BlockID, _after: Option<&Address>, _count: u64) -> Option<Vec<Address>> {
 		None
 	}
 
+	fn list_storage(&self, _id: BlockID, _account: &Address, _after: Option<&H256>, _count: u64) -> Option<Vec<H256>> {
+		None
+	}
 	fn transaction(&self, _id: TransactionID) -> Option<LocalizedTransaction> {
 		None	// Simple default.
 	}

ethcore/src/client/traits.rs

@@ -68,6 +68,10 @@ pub trait BlockChainClient : Sync + Send {
 	/// May not fail on BlockID::Latest.
 	fn nonce(&self, address: &Address, id: BlockID) -> Option<U256>;
 
+	/// Attempt to get address storage root at given block.
+	/// May not fail on BlockID::Latest.
+	fn storage_root(&self, address: &Address, id: BlockID) -> Option<H256>;
+
 	/// Get address nonce at the latest block's state.
 	fn latest_nonce(&self, address: &Address) -> U256 {
 		self.nonce(address, BlockID::Latest)
@@ -114,7 +118,12 @@ pub trait BlockChainClient : Sync + Send {
 	}
 
 	/// Get a list of all accounts in the block `id`, if fat DB is in operation, otherwise `None`.
-	fn list_accounts(&self, id: BlockID) -> Option<Vec<Address>>;
+	/// If `after` is set the list starts with the following item.
+	fn list_accounts(&self, id: BlockID, after: Option<&Address>, count: u64) -> Option<Vec<Address>>;
+
+	/// Get a list of all storage keys in the block `id`, if fat DB is in operation, otherwise `None`.
+	/// If `after` is set the list starts with the following item.
+	fn list_storage(&self, id: BlockID, account: &Address, after: Option<&H256>, count: u64) -> Option<Vec<H256>>;
 
 	/// Get transaction with given hash.
 	fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction>;

ethcore/src/engines/authority_round.rs

@@ -315,6 +315,7 @@ impl Engine for AuthorityRound {
 #[cfg(test)]
 mod tests {
 	use util::*;
+	use util::trie::TrieSpec;
 	use env_info::EnvInfo;
 	use header::Header;
 	use error::{Error, BlockError};
@@ -384,9 +385,9 @@ mod tests {
 		let engine = &*spec.engine;
 		let genesis_header = spec.genesis_header();
 		let mut db1 = get_temp_state_db().take();
-		spec.ensure_db_good(&mut db1).unwrap();
+		spec.ensure_db_good(&mut db1, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let mut db2 = get_temp_state_db().take();
-		spec.ensure_db_good(&mut db2).unwrap();
+		spec.ensure_db_good(&mut db2, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b1 = OpenBlock::new(engine, Default::default(), false, db1, &genesis_header, last_hashes.clone(), addr1, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b1 = b1.close_and_lock();

ethcore/src/engines/basic_authority.rs

@@ -184,6 +184,7 @@ impl Engine for BasicAuthority {
 #[cfg(test)]
 mod tests {
 	use util::*;
+	use util::trie::TrieSpec;
 	use block::*;
 	use env_info::EnvInfo;
 	use error::{BlockError, Error};
@@ -256,7 +257,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/engines/instant_seal.rs

@@ -68,6 +68,7 @@ impl Engine for InstantSeal {
 #[cfg(test)]
 mod tests {
 	use util::*;
+	use util::trie::TrieSpec;
 	use tests::helpers::*;
 	use account_provider::AccountProvider;
 	use spec::Spec;
@@ -84,7 +85,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/ethereum/ethash.rs

@@ -422,6 +422,7 @@ impl Header {
 #[cfg(test)]
 mod tests {
 	use util::*;
+	use util::trie::TrieSpec;
 	use block::*;
 	use tests::helpers::*;
 	use env_info::EnvInfo;
@@ -438,7 +439,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close();
@@ -452,7 +453,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let mut b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle = Header::new();

ethcore/src/ethereum/mod.rs

@@ -72,6 +72,7 @@ pub fn new_morden() -> Spec { load(include_bytes!("../../res/ethereum/morden.jso
 #[cfg(test)]
 mod tests {
 	use util::*;
+	use util::trie::TrieSpec;
 	use state::*;
 	use super::*;
 	use tests::helpers::*;
@@ -84,7 +85,7 @@ mod tests {
 		let genesis_header = spec.genesis_header();
 		let mut db_result = get_temp_state_db();
 		let mut db = db_result.take();
-		spec.ensure_db_good(&mut db).unwrap();
+		spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 		let s = State::from_existing(db, genesis_header.state_root().clone(), engine.account_start_nonce(), Default::default()).unwrap();
 		assert_eq!(s.balance(&"0000000000000000000000000000000000000001".into()), 1u64.into());
 		assert_eq!(s.balance(&"0000000000000000000000000000000000000002".into()), 1u64.into());

ethcore/src/pod_account.rs

@@ -64,13 +64,13 @@ impl PodAccount {
 	}
 
 	/// Place additional data into given hash DB.
-	pub fn insert_additional(&self, db: &mut AccountDBMut) {
+	pub fn insert_additional(&self, db: &mut AccountDBMut, factory: &TrieFactory) {
 		match self.code {
 			Some(ref c) if !c.is_empty() => { db.insert(c); }
 			_ => {}
 		}
 		let mut r = H256::new();
-		let mut t = SecTrieDBMut::new(db, &mut r);
+		let mut t = factory.create(db, &mut r);
 		for (k, v) in &self.storage {
 			if let Err(e) = t.insert(k, &rlp::encode(&U256::from(&**v))) {
 				warn!("Encountered potential DB corruption: {}", e);

ethcore/src/spec/spec.rs

@@ -244,13 +244,13 @@ impl Spec {
 	}
 
 	/// Ensure that the given state DB has the trie nodes in for the genesis state.
-	pub fn ensure_db_good(&self, db: &mut StateDB) -> Result<bool, Box<TrieError>> {
+	pub fn ensure_db_good(&self, db: &mut StateDB, factory: &TrieFactory) -> Result<bool, Box<TrieError>> {
 		if !db.as_hashdb().contains(&self.state_root()) {
 			trace!(target: "spec", "ensure_db_good: Fresh database? Cannot find state root {}", self.state_root());
 			let mut root = H256::new();
 
 			{
-				let mut t = SecTrieDBMut::new(db.as_hashdb_mut(), &mut root);
+				let mut t = factory.create(db.as_hashdb_mut(), &mut root);
 				for (address, account) in self.genesis_state.get().iter() {
 					try!(t.insert(&**address, &account.rlp()));
 				}
@@ -258,7 +258,7 @@ impl Spec {
 			trace!(target: "spec", "ensure_db_good: Populated sec trie; root is {}", root);
 			for (address, account) in self.genesis_state.get().iter() {
 				db.note_non_null_account(address);
-				account.insert_additional(&mut AccountDBMut::new(db.as_hashdb_mut(), address));
+				account.insert_additional(&mut AccountDBMut::new(db.as_hashdb_mut(), address), factory);
 			}
 			assert!(db.as_hashdb().contains(&self.state_root()));
 			Ok(true)

ethcore/src/state/account.rs

@@ -314,11 +314,10 @@ impl Account {
 		self.code_hash == SHA3_EMPTY
 	}
 
-	#[cfg(test)]
-	/// return the storage root associated with this account or None if it has been altered via the overlay.
+	/// Return the storage root associated with this account or None if it has been altered via the overlay.
 	pub fn storage_root(&self) -> Option<&H256> { if self.storage_is_clean() {Some(&self.storage_root)} else {None} }
 
-	/// return the storage overlay.
+	/// Return the storage overlay.
 	pub fn storage_changes(&self) -> &HashMap<H256, H256> { &self.storage_changes }
 
 	/// Increment the nonce of the account by one.

ethcore/src/state/mod.rs

@@ -369,6 +369,12 @@ impl State {
 			|a| a.as_ref().map_or(self.account_start_nonce, |account| *account.nonce()))
 	}
 
+	/// Get the storage root of account `a`.
+	pub fn storage_root(&self, a: &Address) -> Option<H256> {
+		self.ensure_cached(a, RequireCache::None, true,
+			|a| a.as_ref().and_then(|account| account.storage_root().cloned()))
+	}
+
 	/// Mutate storage of account `address` so that it is `value` for `key`.
 	pub fn storage_at(&self, address: &Address, key: &H256) -> H256 {
 		// Storage key search and update works like this:

ethcore/src/tests/helpers.rs

@@ -18,6 +18,7 @@ use ethkey::KeyPair;
 use io::*;
 use client::{BlockChainClient, Client, ClientConfig};
 use util::*;
+use util::trie::TrieSpec;
 use spec::*;
 use state_db::StateDB;
 use block::{OpenBlock, Drain};
@@ -157,7 +158,7 @@ pub fn generate_dummy_client_with_spec_and_data<F>(get_test_spec: F, block_numbe
 
 	let mut db_result = get_temp_state_db();
 	let mut db = db_result.take();
-	test_spec.ensure_db_good(&mut db).unwrap();
+	test_spec.ensure_db_good(&mut db, &TrieFactory::new(TrieSpec::Secure)).unwrap();
 	let genesis_header = test_spec.genesis_header();
 
 	let mut rolling_timestamp = 40;