Add a optional json dump state to evm-bin (#9706)

* Add a dump of the state at the end of transaction for --json-test.
Also fixes json-test output on finish, and allow to put both on err or
out (--out-only and --err-only).

* Dump state resolution from trie, function behind evm-bin feature to avoid misuse.

* Rename 'slow' method to 'to_pod_full'.
Use cache first in 'to_pod_full', for in between commits case.
Change dump activation to use a function pointer instead.

* Fix tests.

* Query and add storage values to dump.

* Switch to use `require` method, even if less efficient it is better in
this case to reuse existing code.
Reuse of `storage_at` was not easy in this case (could not iterate and
use the method at the same time (refcell mutable borrow panics) so
keeping code as is.

* Switch to returning error.
Use 'base_storage_root' instead of 'storage_root'.
Added a test, it will only execute with json-test in ci, or when launch
with the feature.

* Renaming of command line parameters.
Comments fixes.
Minor code changes.

* Fix evmbin cmd parsing test.

* README update.

* Fix extra space and avoid clone call on copiable address.

* Revert test submodule.

* Revert wasm-test submodule.

* Use map_or instead of map + unwrap_or

* restore tests submodule

ethcore/src/state/account.rs

@@ -624,7 +624,7 @@ mod tests {
 		assert!(raw.len() > compact_vec.len());
 		let again_raw = decompress(&compact_vec, snapshot_swapper());
 		assert_eq!(raw, again_raw.into_vec());
-    }
+	}
 
 	#[test]
 	fn storage_at() {

ethcore/src/state/mod.rs

@@ -947,20 +947,78 @@ impl<B: Backend> State<B> {
 	}
 
 	/// Populate a PodAccount map from this state.
-	pub fn to_pod(&self) -> PodState {
+	fn to_pod_cache(&self) -> PodState {
 		assert!(self.checkpoints.borrow().is_empty());
-		// TODO: handle database rather than just the cache.
-		// will need fat db.
 		PodState::from(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| {
 			if let Some(ref acc) = opt.account {
-				m.insert(add.clone(), PodAccount::from_account(acc));
+				m.insert(*add, PodAccount::from_account(acc));
 			}
 			m
 		}))
 	}
 
+	#[cfg(feature="to-pod-full")]
+	/// Populate a PodAccount map from this state.
+	/// Warning this is not for real time use.
+	/// Use of this method requires FatDB mode to be able 
+	/// to iterate on accounts.
+	pub fn to_pod_full(&self) -> Result<PodState, Error> {
+
+		assert!(self.checkpoints.borrow().is_empty());
+		assert!(self.factories.trie.is_fat());
+
+		let mut result = BTreeMap::new();
+
+		let trie = self.factories.trie.readonly(self.db.as_hashdb(), &self.root)?;
+
+		// put trie in cache 
+		for item in trie.iter()? {
+			if let Ok((addr, _dbval)) = item {
+				let address = Address::from_slice(&addr);
+				let _ = self.require(&address, true);
+			}
+		}
+
+		// Resolve missing part
+		for (add, opt) in self.cache.borrow().iter() {
+			if let Some(ref acc) = opt.account {
+				let pod_account = self.account_to_pod_account(acc, add)?;
+				result.insert(add.clone(), pod_account);
+			}
+		}
+
+		Ok(PodState::from(result))
+	}
+
+	/// Create a PodAccount from an account.
+	/// Differs from existing method by including all storage
+	/// values of the account to the PodAccount.
+	/// This function is only intended for use in small tests or with fresh accounts.
+	/// It requires FatDB.
+	#[cfg(feature="to-pod-full")]
+	fn account_to_pod_account(&self, account: &Account, address: &Address) -> Result<PodAccount, Error> {
+		let mut pod_storage = BTreeMap::new();
+		let addr_hash = account.address_hash(address);
+		let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash);
+		let root = account.base_storage_root();
+
+		let trie = self.factories.trie.readonly(accountdb.as_hashdb(), &root)?;
+		for o_kv in trie.iter()? {
+			if let Ok((key, val)) = o_kv {
+				pod_storage.insert(key[..].into(), U256::from(&val[..]).into());
+			}
+		}
+
+		let mut pod_account = PodAccount::from_account(&account);
+		// cached one first
+		pod_storage.append(&mut pod_account.storage);
+		pod_account.storage = pod_storage;
+		Ok(pod_account)
+	}
+
+
 	/// Populate a PodAccount map from this state, with another state as the account and storage query.
-	pub fn to_pod_diff<X: Backend>(&mut self, query: &State<X>) -> TrieResult<PodState> {
+	fn to_pod_diff<X: Backend>(&mut self, query: &State<X>) -> TrieResult<PodState> {
 		assert!(self.checkpoints.borrow().is_empty());
 
 		// Merge PodAccount::to_pod for cache of self and `query`.
@@ -1015,7 +1073,7 @@ impl<B: Backend> State<B> {
 	/// Returns a `StateDiff` describing the difference from `orig` to `self`.
 	/// Consumes self.
 	pub fn diff_from<X: Backend>(&self, mut orig: State<X>) -> TrieResult<StateDiff> {
-		let pod_state_post = self.to_pod();
+		let pod_state_post = self.to_pod_cache();
 		let pod_state_pre = orig.to_pod_diff(self)?;
 		Ok(pod_state::diff_pod(&pod_state_pre, &pod_state_post))
 	}
@@ -2593,12 +2651,12 @@ mod tests {
 		assert_eq!(diff_map.len(), 1);
 		assert!(diff_map.get(&a).is_some());
 		assert_eq!(diff_map.get(&a),
-				   pod_account::diff_pod(Some(&PodAccount {
-					   balance: U256::from(100),
-					   nonce: U256::zero(),
-					   code: Some(Default::default()),
-					   storage: Default::default()
-				   }), None).as_ref());
+				pod_account::diff_pod(Some(&PodAccount {
+					balance: U256::from(100),
+					nonce: U256::zero(),
+					code: Some(Default::default()),
+					storage: Default::default()
+				}), None).as_ref());
 	}
 
 	#[test]
@@ -2624,18 +2682,64 @@ mod tests {
 		assert_eq!(diff_map.len(), 1);
 		assert!(diff_map.get(&a).is_some());
 		assert_eq!(diff_map.get(&a),
-				   pod_account::diff_pod(Some(&PodAccount {
-					   balance: U256::zero(),
-					   nonce: U256::zero(),
-					   code: Some(Default::default()),
-					   storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(20u64)))]
-						   .into_iter().collect(),
-				   }), Some(&PodAccount {
-					   balance: U256::zero(),
-					   nonce: U256::zero(),
-					   code: Some(Default::default()),
-					   storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(100u64)))]
-						   .into_iter().collect(),
-				   })).as_ref());
+				pod_account::diff_pod(Some(&PodAccount {
+					balance: U256::zero(),
+					nonce: U256::zero(),
+					code: Some(Default::default()),
+					storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(20u64)))]
+						.into_iter().collect(),
+				}), Some(&PodAccount {
+					balance: U256::zero(),
+					nonce: U256::zero(),
+					code: Some(Default::default()),
+					storage: vec![(H256::from(&U256::from(1u64)), H256::from(&U256::from(100u64)))]
+						.into_iter().collect(),
+				})).as_ref());
 	}
+
+	#[cfg(feature="to-pod-full")]
+	#[test]
+	fn should_get_full_pod_storage_values() {
+		use trie::{TrieFactory, TrieSpec};
+
+		let a = 10.into();
+		let db = get_temp_state_db();
+
+		let factories = Factories {
+			vm: Default::default(),
+			trie: TrieFactory::new(TrieSpec::Fat),
+			accountdb: Default::default(),
+		};
+
+		let get_pod_state_val = |pod_state : &PodState, ak, k| {
+			pod_state.get().get(ak).unwrap().storage.get(&k).unwrap().clone()
+		};
+
+		let storage_address = H256::from(&U256::from(1u64));
+
+		let (root, db) = {
+			let mut state = State::new(db, U256::from(0), factories.clone());
+			state.set_storage(&a, storage_address.clone(), H256::from(&U256::from(20u64))).unwrap();
+			let dump = state.to_pod_full().unwrap();
+			assert_eq!(get_pod_state_val(&dump, &a, storage_address.clone()), H256::from(&U256::from(20u64)));
+			state.commit().unwrap();
+			let dump = state.to_pod_full().unwrap();
+			assert_eq!(get_pod_state_val(&dump, &a, storage_address.clone()), H256::from(&U256::from(20u64)));
+			state.drop()
+		};
+
+		let mut state = State::from_existing(db, root, U256::from(0u8), factories).unwrap();
+		let dump = state.to_pod_full().unwrap();
+		assert_eq!(get_pod_state_val(&dump, &a, storage_address.clone()), H256::from(&U256::from(20u64)));
+		state.set_storage(&a, storage_address.clone(), H256::from(&U256::from(21u64))).unwrap();
+		let dump = state.to_pod_full().unwrap();
+		assert_eq!(get_pod_state_val(&dump, &a, storage_address.clone()), H256::from(&U256::from(21u64)));
+		state.commit().unwrap();
+		state.set_storage(&a, storage_address.clone(), H256::from(&U256::from(0u64))).unwrap();
+		let dump = state.to_pod_full().unwrap();
+		assert_eq!(get_pod_state_val(&dump, &a, storage_address.clone()), H256::from(&U256::from(0u64)));
+
+
+	}
+
 }