[stable] Backports (#5297)

* v1.5.12

* Fine grained snapshot chunking

Cargo.lock

@@ -1,6 +1,6 @@
 [root]
 name = "parity"
-version = "1.5.11"
+version = "1.5.12"
 dependencies = [
  "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -23,7 +23,7 @@ dependencies = [
  "ethcore-rpc 1.5.0",
  "ethcore-signer 1.5.0",
  "ethcore-stratum 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethsync 1.5.0",
  "evmbin 1.5.0",
  "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -303,6 +303,11 @@ name = "dtoa"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "either"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "elastic-array"
 version = "0.6.0"
@@ -373,13 +378,14 @@ dependencies = [
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
  "ethcore-ipc-nano 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethjson 0.1.0",
  "ethkey 0.2.0",
  "ethstore 0.1.0",
  "evmjit 1.5.0",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)",
+ "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -421,7 +427,7 @@ dependencies = [
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-devtools 1.5.0",
  "ethcore-rpc 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "fetch 0.1.0",
  "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)",
@@ -469,7 +475,7 @@ name = "ethcore-ipc"
 version = "1.5.0"
 dependencies = [
  "ethcore-devtools 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
  "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -516,7 +522,7 @@ dependencies = [
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
  "ethcore-ipc-nano 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
  "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -531,7 +537,7 @@ dependencies = [
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
  "ethcore-network 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlp 0.1.0",
  "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -543,7 +549,7 @@ name = "ethcore-logger"
 version = "1.5.0"
 dependencies = [
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -559,7 +565,7 @@ dependencies = [
  "bytes 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-devtools 1.5.0",
  "ethcore-io 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethcrypto 0.1.0",
  "ethkey 0.2.0",
  "igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -586,7 +592,7 @@ dependencies = [
  "ethcore-devtools 1.5.0",
  "ethcore-io 1.5.0",
  "ethcore-ipc 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethcrypto 0.1.0",
  "ethjson 0.1.0",
  "ethkey 0.2.0",
@@ -621,7 +627,7 @@ dependencies = [
  "ethcore-devtools 1.5.0",
  "ethcore-io 1.5.0",
  "ethcore-rpc 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-dapps-glue 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -640,7 +646,7 @@ dependencies = [
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
  "ethcore-ipc-nano 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)",
  "jsonrpc-tcp-server 0.1.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -651,7 +657,7 @@ dependencies = [
 
 [[package]]
 name = "ethcore-util"
-version = "1.5.11"
+version = "1.5.12"
 dependencies = [
  "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -663,7 +669,7 @@ dependencies = [
  "ethcore-bloom-journal 0.1.0",
  "ethcore-devtools 1.5.0",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -700,7 +706,7 @@ dependencies = [
 name = "ethjson"
 version = "0.1.0"
 dependencies = [
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -727,7 +733,7 @@ dependencies = [
  "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcrypto 0.1.0",
  "ethkey 0.2.0",
- "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -756,7 +762,7 @@ dependencies = [
  "ethcore-ipc-nano 1.5.0",
  "ethcore-light 1.5.0",
  "ethcore-network 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethkey 0.2.0",
  "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -773,7 +779,7 @@ version = "1.5.0"
 dependencies = [
  "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -975,7 +981,7 @@ version = "1.5.0"
 dependencies = [
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -992,9 +998,17 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.4.13"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "itertools"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "itoa"
 version = "0.1.1"
@@ -1500,7 +1514,7 @@ name = "parity-hash-fetch"
 version = "1.5.0"
 dependencies = [
  "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "fetch 0.1.0",
  "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1525,7 +1539,7 @@ version = "1.4.0"
 dependencies = [
  "ethcore-rpc 1.5.0",
  "ethcore-signer 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1571,7 +1585,7 @@ dependencies = [
  "ethcore 1.5.0",
  "ethcore-ipc 1.5.0",
  "ethcore-ipc-codegen 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "ethsync 1.5.0",
  "ipc-common-types 1.5.0",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1851,7 +1865,7 @@ version = "1.4.0"
 dependencies = [
  "ethcore-bigint 0.1.2",
  "ethcore-rpc 1.5.0",
- "ethcore-util 1.5.11",
+ "ethcore-util 1.5.12",
  "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-rpc-client 1.4.0",
  "rpassword 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2469,6 +2483,7 @@ dependencies = [
 "checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76"
 "checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d"
 "checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f"
+"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
 "checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "<none>"
 "checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5"
 "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
@@ -2491,7 +2506,8 @@ dependencies = [
 "checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
 "checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484"
 "checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c"
-"checksum itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "086e1fa5fe48840b1cfdef3a20c7e3115599f8d5c4c87ef32a794a7cdd184d76"
+"checksum itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)" = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"
+"checksum itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d95557e7ba6b71377b0f2c3b3ae96c53f1b75a926a6901a500f557a370af730a"
 "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
 "checksum itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91fd9dc2c587067de817fec4ad355e3818c3d893a78cab32a0a474c7a15bb8d5"
 "checksum jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)" = "<none>"

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 description = "Parity Ethereum client"
 name = "parity"
-version = "1.5.11"
+version = "1.5.12"
 license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"

ethcore/Cargo.toml

@@ -28,6 +28,7 @@ rand = "0.3"
 byteorder = "0.5"
 transient-hashmap = "0.1"
 linked-hash-map = "0.3.0"
+itertools = "0.5"
 evmjit = { path = "../evmjit", optional = true }
 clippy = { version = "0.0.103", optional = true}
 ethash = { path = "../ethash" }

ethcore/src/lib.rs

@@ -103,6 +103,7 @@ extern crate ethcore_bloom_journal as bloom_journal;
 extern crate byteorder;
 extern crate transient_hashmap;
 extern crate linked_hash_map;
+extern crate itertools;
 
 #[macro_use]
 extern crate log;

ethcore/src/snapshot/account.rs

@@ -23,6 +23,7 @@ use snapshot::Error;
 use util::{U256, FixedHash, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP};
 use util::trie::{TrieDB, Trie};
 use rlp::{RlpStream, Stream, UntrustedRlp, View};
+use itertools::Itertools;
 
 use std::collections::HashSet;
 
@@ -60,55 +61,53 @@ impl CodeState {
 	}
 }
 
-// walk the account's storage trie, returning an RLP item containing the
+// walk the account's storage trie, returning a vector of RLP items containing the
-// account properties and the storage.
+// account properties and the storage. Each item contains at most `max_storage_items`
-pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet<H256>) -> Result<Bytes, Error> {
+// storage records split according to snapshot format definition.
+pub fn to_fat_rlps(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet<H256>, max_storage_items: usize) -> Result<Vec<Bytes>, Error> {
 	if acc == &ACC_EMPTY {
-		return Ok(::rlp::NULL_RLP.to_vec());
+		return Ok(vec![::rlp::NULL_RLP.to_vec()]);
 	}
 
 	let db = TrieDB::new(acct_db, &acc.storage_root)?;
 
-	let mut pairs = Vec::new();
+	let chunks = db.iter()?.chunks(max_storage_items);
+	let pair_chunks = chunks.into_iter().map(|chunk| chunk.collect());
+	pair_chunks.pad_using(1, |_| Vec::new(), ).map(|pairs| {
+		let mut stream = RlpStream::new_list(pairs.len());
 
-	for item in db.iter()? {
+		for r in pairs {
-		let (k, v) = item?;
+			let (k, v) = r?;
-		pairs.push((k, v));
+			stream.begin_list(2).append(&k).append(&&*v);
-	}
+		}
 
-	let mut stream = RlpStream::new_list(pairs.len());
+		let pairs_rlp = stream.out();
 
-	for (k, v) in pairs {
+		let mut account_stream = RlpStream::new_list(5);
-		stream.begin_list(2).append(&k).append(&&*v);
+		account_stream.append(&acc.nonce)
-	}
+					  .append(&acc.balance);
 
-	let pairs_rlp = stream.out();
+		// [has_code, code_hash].
-
+		if acc.code_hash == SHA3_EMPTY {
-	let mut account_stream = RlpStream::new_list(5);
+			account_stream.append(&CodeState::Empty.raw()).append_empty_data();
-	account_stream.append(&acc.nonce)
+		} else if used_code.contains(&acc.code_hash) {
-				  .append(&acc.balance);
+			account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash);
-
+		} else {
-	// [has_code, code_hash].
+			match acct_db.get(&acc.code_hash) {
-	if acc.code_hash == SHA3_EMPTY {
+				Some(c) => {
-		account_stream.append(&CodeState::Empty.raw()).append_empty_data();
+					used_code.insert(acc.code_hash.clone());
-	} else if used_code.contains(&acc.code_hash) {
+					account_stream.append(&CodeState::Inline.raw()).append(&&*c);
-		account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash);
+				}
-	} else {
+				None => {
-		match acct_db.get(&acc.code_hash) {
+					warn!("code lookup failed during snapshot");
-			Some(c) => {
+					account_stream.append(&false).append_empty_data();
-				used_code.insert(acc.code_hash.clone());
+				}
-				account_stream.append(&CodeState::Inline.raw()).append(&&*c);
-			}
-			None => {
-				warn!("code lookup failed during snapshot");
-				account_stream.append(&false).append_empty_data();
 			}
 		}
-	}
 
-	account_stream.append_raw(&pairs_rlp, 1);
+		account_stream.append_raw(&pairs_rlp, 1);
-
+		Ok(account_stream.out())
-	Ok(account_stream.out())
+	}).collect()
 }
 
 // decode a fat rlp, and rebuild the storage trie as we go.
@@ -117,6 +116,7 @@ pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashS
 pub fn from_fat_rlp(
 	acct_db: &mut AccountDBMut,
 	rlp: UntrustedRlp,
+	mut storage_root: H256,
 ) -> Result<(BasicAccount, Option<Bytes>), Error> {
 	use util::{TrieDBMut, TrieMut};
 
@@ -148,10 +148,12 @@ pub fn from_fat_rlp(
 		}
 	};
 
-	let mut storage_root = H256::zero();
-
 	{
-		let mut storage_trie = TrieDBMut::new(acct_db, &mut storage_root);
+		let mut storage_trie = if storage_root.is_zero() {
+			TrieDBMut::new(acct_db, &mut storage_root)
+		} else {
+			TrieDBMut::from_existing(acct_db, &mut storage_root)?
+		};
 		let pairs = rlp.at(4)?;
 		for pair_rlp in pairs.iter() {
 			let k: Bytes  = pair_rlp.val_at(0)?;
@@ -184,7 +186,7 @@ mod tests {
 
 	use std::collections::HashSet;
 
-	use super::{ACC_EMPTY, to_fat_rlp, from_fat_rlp};
+	use super::{ACC_EMPTY, to_fat_rlps, from_fat_rlp};
 
 	#[test]
 	fn encoding_basic() {
@@ -201,9 +203,9 @@ mod tests {
 		let thin_rlp = ::rlp::encode(&account);
 		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
 
-		let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap();
+		let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap();
-		let fat_rlp = UntrustedRlp::new(&fat_rlp);
+		let fat_rlp = UntrustedRlp::new(&fat_rlps[0]);
-		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account);
+		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account);
 	}
 
 	#[test]
@@ -226,9 +228,40 @@ mod tests {
 		let thin_rlp = ::rlp::encode(&account);
 		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
 
-		let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap();
+		let fat_rlp = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap();
-		let fat_rlp = UntrustedRlp::new(&fat_rlp);
+		let fat_rlp = UntrustedRlp::new(&fat_rlp[0]);
-		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account);
+		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account);
+	}
+
+	#[test]
+	fn encoding_storage_split() {
+		let mut db = get_temp_state_db();
+		let addr = Address::random();
+
+		let account = {
+			let acct_db = AccountDBMut::new(db.as_hashdb_mut(), &addr);
+			let mut root = SHA3_NULL_RLP;
+			fill_storage(acct_db, &mut root, &mut H256::zero());
+			BasicAccount {
+				nonce: 25.into(),
+				balance: 987654321.into(),
+				storage_root: root,
+				code_hash: SHA3_EMPTY,
+			}
+		};
+
+		let thin_rlp = ::rlp::encode(&account);
+		assert_eq!(::rlp::decode::<BasicAccount>(&thin_rlp), account);
+
+		let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), 100).unwrap();
+		let mut root = SHA3_NULL_RLP;
+		let mut restored_account = None;
+		for rlp in fat_rlps {
+			let fat_rlp = UntrustedRlp::new(&rlp);
+			restored_account = Some(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, root).unwrap().0);
+			root = restored_account.as_ref().unwrap().storage_root.clone();
+		}
+		assert_eq!(restored_account, Some(account));
 	}
 
 	#[test]
@@ -264,18 +297,18 @@ mod tests {
 
 		let mut used_code = HashSet::new();
 
-		let fat_rlp1 = to_fat_rlp(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code).unwrap();
+		let fat_rlp1 = to_fat_rlps(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code, usize::max_value()).unwrap();
-		let fat_rlp2 = to_fat_rlp(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code).unwrap();
+		let fat_rlp2 = to_fat_rlps(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code, usize::max_value()).unwrap();
 		assert_eq!(used_code.len(), 1);
 
-		let fat_rlp1 = UntrustedRlp::new(&fat_rlp1);
+		let fat_rlp1 = UntrustedRlp::new(&fat_rlp1[0]);
-		let fat_rlp2 = UntrustedRlp::new(&fat_rlp2);
+		let fat_rlp2 = UntrustedRlp::new(&fat_rlp2[0]);
 
-		let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2).unwrap();
+		let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2, H256::zero()).unwrap();
 		assert!(maybe_code.is_none());
 		assert_eq!(acc, account2);
 
-		let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1).unwrap();
+		let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1, H256::zero()).unwrap();
 		assert_eq!(maybe_code, Some(b"this is definitely code".to_vec()));
 		assert_eq!(acc, account1);
 	}
@@ -285,7 +318,7 @@ mod tests {
 		let mut db = get_temp_state_db();
 		let mut used_code = HashSet::new();
 
-		assert_eq!(to_fat_rlp(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code).unwrap(), ::rlp::NULL_RLP.to_vec());
+		assert_eq!(to_fat_rlps(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code, usize::max_value()).unwrap(), vec![::rlp::NULL_RLP.to_vec()]);
-		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP)).unwrap(), (ACC_EMPTY, None));
+		assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP), H256::zero()).unwrap(), (ACC_EMPTY, None));
 	}
 }

ethcore/src/snapshot/error.rs

@@ -53,6 +53,8 @@ pub enum Error {
 	Decoder(DecoderError),
 	/// Io error.
 	Io(::std::io::Error),
+	/// Snapshot version is not supported.
+	VersionNotSupported(u64),
 }
 
 impl fmt::Display for Error {
@@ -73,6 +75,7 @@ impl fmt::Display for Error {
 			Error::Io(ref err) => err.fmt(f),
 			Error::Decoder(ref err) => err.fmt(f),
 			Error::Trie(ref err) => err.fmt(f),
+			Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver),
 		}
 	}
 }

ethcore/src/snapshot/io.rs

@@ -31,6 +31,8 @@ use rlp::{self, Encodable, RlpStream, UntrustedRlp, Stream, View};
 
 use super::ManifestData;
 
+const SNAPSHOT_VERSION: u64 = 2;
+
 /// Something which can write snapshots.
 /// Writing the same chunk multiple times will lead to implementation-defined
 /// behavior, and is not advised.
@@ -120,8 +122,9 @@ impl SnapshotWriter for PackedWriter {
 	fn finish(mut self, manifest: ManifestData) -> io::Result<()> {
 		// we ignore the hashes fields of the manifest under the assumption that
 		// they are consistent with ours.
-		let mut stream = RlpStream::new_list(5);
+		let mut stream = RlpStream::new_list(6);
 		stream
+			.append(&SNAPSHOT_VERSION)
 			.append(&self.state_hashes)
 			.append(&self.block_hashes)
 			.append(&manifest.state_root)
@@ -223,7 +226,7 @@ impl PackedReader {
 	/// Create a new `PackedReader` for the file at the given path.
 	/// This will fail if any io errors are encountered or the file
 	/// is not a valid packed snapshot.
-	pub fn new(path: &Path) -> Result<Option<Self>, ::error::Error> {
+	pub fn new(path: &Path) -> Result<Option<Self>, ::snapshot::error::Error> {
 		let mut file = File::open(path)?;
 		let file_len = file.metadata()?.len();
 		if file_len < 8 {
@@ -257,15 +260,26 @@ impl PackedReader {
 
 		let rlp = UntrustedRlp::new(&manifest_buf);
 
-		let state: Vec<ChunkInfo> = rlp.val_at(0)?;
+		let (start, version) = if rlp.item_count() == 5 {
-		let blocks: Vec<ChunkInfo> = rlp.val_at(1)?;
+			(0, 1)
+		} else {
+			(1, rlp.val_at(0)?)
+		};
+
+		if version > SNAPSHOT_VERSION {
+			return Err(::snapshot::error::Error::VersionNotSupported(version));
+		}
+
+		let state: Vec<ChunkInfo> = rlp.val_at(0 + start)?;
+		let blocks: Vec<ChunkInfo> = rlp.val_at(1 + start)?;
 
 		let manifest = ManifestData {
+			version: version,
 			state_hashes: state.iter().map(|c| c.0).collect(),
 			block_hashes: blocks.iter().map(|c| c.0).collect(),
-			state_root: rlp.val_at(2)?,
+			state_root: rlp.val_at(2 + start)?,
-			block_number: rlp.val_at(3)?,
+			block_number: rlp.val_at(3 + start)?,
-			block_hash: rlp.val_at(4)?,
+			block_hash: rlp.val_at(4 + start)?,
 		};
 
 		Ok(Some(PackedReader {
@@ -348,7 +362,7 @@ mod tests {
 	use util::sha3::Hashable;
 
 	use snapshot::ManifestData;
-	use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader};
+	use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION};
 
 	const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"];
 	const BLOCK_CHUNKS: &'static [&'static [u8]] = &[b"hello!", b"goodbye!", b"abcdefg", b"hijklmnop", b"qrstuvwxy", b"and", b"z"];
@@ -374,6 +388,7 @@ mod tests {
 		}
 
 		let manifest = ManifestData {
+			version: SNAPSHOT_VERSION,
 			state_hashes: state_hashes,
 			block_hashes: block_hashes,
 			state_root: b"notarealroot".sha3(),
@@ -412,6 +427,7 @@ mod tests {
 		}
 
 		let manifest = ManifestData {
+			version: SNAPSHOT_VERSION,
 			state_hashes: state_hashes,
 			block_hashes: block_hashes,
 			state_root: b"notarealroot".sha3(),

ethcore/src/snapshot/mod.rs

@@ -56,6 +56,7 @@ pub use self::traits::SnapshotService;
 pub use self::watcher::Watcher;
 pub use types::snapshot_manifest::ManifestData;
 pub use types::restoration_status::RestorationStatus;
+pub use types::basic_account::BasicAccount;
 
 pub mod io;
 pub mod service;
@@ -82,6 +83,9 @@ mod traits {
 // Try to have chunks be around 4MB (before compression)
 const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
 
+// Try to have chunks be around 4MB (before compression)
+const MAX_STORAGE_ENTRIES_PER_ACCOUNT_RECORD: usize = 80_000;
+
 // How many blocks to include in a snapshot, starting from the head of the chain.
 const SNAPSHOT_BLOCKS: u64 = 30000;
 
@@ -147,6 +151,7 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
 	info!("produced {} state chunks and {} block chunks.", state_hashes.len(), block_hashes.len());
 
 	let manifest_data = ManifestData {
+		version: 2,
 		state_hashes: state_hashes,
 		block_hashes: block_hashes,
 		state_root: *state_root,
@@ -300,14 +305,14 @@ impl<'a> StateChunker<'a> {
 	//
 	// If the buffer is greater than the desired chunk size,
 	// this will write out the data to disk.
-	fn push(&mut self, account_hash: Bytes, data: Bytes) -> Result<(), Error> {
+	fn push(&mut self, account_hash: Bytes, data: Bytes, force_chunk: bool) -> Result<(), Error> {
 		let pair = {
 			let mut stream = RlpStream::new_list(2);
 			stream.append(&account_hash).append_raw(&data, 1);
 			stream.out()
 		};
 
-		if self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE {
+		if force_chunk || self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE {
 			self.write_chunk()?;
 		}
 
@@ -372,8 +377,10 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
 
 		let account_db = AccountDB::from_hash(db, account_key_hash);
 
-		let fat_rlp = account::to_fat_rlp(&account, &account_db, &mut used_code)?;
+		let fat_rlps = account::to_fat_rlps(&account, &account_db, &mut used_code, MAX_STORAGE_ENTRIES_PER_ACCOUNT_RECORD)?;
-		chunker.push(account_key, fat_rlp)?;
+		for (i, fat_rlp) in fat_rlps.into_iter().enumerate() {
+			chunker.push(account_key.clone(), fat_rlp, i > 0)?;
+		}
 	}
 
 	if chunker.cur_size != 0 {
@@ -390,6 +397,7 @@ pub struct StateRebuilder {
 	known_code: HashMap<H256, H256>, // code hashes mapped to first account with this code.
 	missing_code: HashMap<H256, Vec<H256>>, // maps code hashes to lists of accounts missing that code.
 	bloom: Bloom,
+	known_storage_roots: HashMap<H256, H256>, // maps account hashes to last known storage root. Only filled for last account per chunk.
 }
 
 impl StateRebuilder {
@@ -401,6 +409,7 @@ impl StateRebuilder {
 			known_code: HashMap::new(),
 			missing_code: HashMap::new(),
 			bloom: StateDB::load_bloom(&*db),
+			known_storage_roots: HashMap::new(),
 		}
 	}
 
@@ -418,6 +427,7 @@ impl StateRebuilder {
 			rlp,
 			&mut pairs,
 			&self.known_code,
+			&mut self.known_storage_roots,
 			flag
 		)?;
 
@@ -492,10 +502,11 @@ fn rebuild_accounts(
 	account_fat_rlps: UntrustedRlp,
 	out_chunk: &mut [(H256, Bytes)],
 	known_code: &HashMap<H256, H256>,
+	known_storage_roots: &mut HashMap<H256, H256>,
 	abort_flag: &AtomicBool,
 ) -> Result<RebuiltStatus, ::error::Error> {
 	let mut status = RebuiltStatus::default();
-	for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk) {
+	for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk.iter_mut()) {
 		if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) }
 
 		let hash: H256 = account_rlp.val_at(0)?;
@@ -506,7 +517,8 @@ fn rebuild_accounts(
 			// fill out the storage trie and code while decoding.
 			let (acc, maybe_code) = {
 				let mut acct_db = AccountDBMut::from_hash(db, hash);
-				account::from_fat_rlp(&mut acct_db, fat_rlp)?
+				let storage_root = known_storage_roots.get(&hash).cloned().unwrap_or(H256::zero());
+				account::from_fat_rlp(&mut acct_db, fat_rlp, storage_root)?
 			};
 
 			let code_hash = acc.code_hash.clone();
@@ -538,6 +550,12 @@ fn rebuild_accounts(
 
 		*out = (hash, thin_rlp);
 	}
+	if let Some(&(ref hash, ref rlp)) = out_chunk.iter().last() {
+		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp).storage_root);
+	}
+	if let Some(&(ref hash, ref rlp)) = out_chunk.iter().next() {
+		known_storage_roots.insert(*hash, ::rlp::decode::<BasicAccount>(rlp).storage_root);
+	}
 	Ok(status)
 }
 

ethcore/src/snapshot/service.rs

@@ -658,6 +658,7 @@ mod tests {
 		assert_eq!(service.status(), RestorationStatus::Inactive);
 
 		let manifest = ManifestData {
+			version: 2,
 			state_hashes: vec![],
 			block_hashes: vec![],
 			state_root: Default::default(),

ethcore/src/snapshot/tests/blocks.rs

@@ -63,6 +63,7 @@ fn chunk_and_restore(amount: u64) {
 	let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap());
 	let block_hashes = chunk_blocks(&bc, best_hash, &writer, &Progress::default()).unwrap();
 	let manifest = ::snapshot::ManifestData {
+		version: 2,
 		state_hashes: Vec::new(),
 		block_hashes: block_hashes,
 		state_root: ::util::sha3::SHA3_NULL_RLP,
@@ -125,6 +126,7 @@ fn checks_flag() {
 	let chain = BlockChain::new(Default::default(), &genesis, db.clone(), engine.clone());
 
 	let manifest = ::snapshot::ManifestData {
+		version: 2,
 		state_hashes: Vec::new(),
 		block_hashes: Vec::new(),
 		state_root: ::util::sha3::SHA3_NULL_RLP,

ethcore/src/snapshot/tests/mod.rs

@@ -27,6 +27,7 @@ use super::ManifestData;
 #[test]
 fn manifest_rlp() {
 	let manifest = ManifestData {
+		version: 2,
 		block_hashes: Vec::new(),
 		state_hashes: Vec::new(),
 		block_number: 1234567,

ethcore/src/snapshot/tests/service.rs

@@ -122,6 +122,7 @@ fn guards_delete_folders() {
 	path.push("restoration");
 
 	let manifest = ManifestData {
+		version: 2,
 		state_hashes: vec![],
 		block_hashes: vec![],
 		block_number: 0,

ethcore/src/snapshot/tests/state.rs

@@ -58,6 +58,7 @@ fn snap_and_restore() {
 	let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap();
 
 	writer.into_inner().finish(::snapshot::ManifestData {
+		version: 2,
 		state_hashes: state_hashes,
 		block_hashes: Vec::new(),
 		state_root: state_root,
@@ -120,10 +121,10 @@ fn get_code_from_prev_chunk() {
 		let mut db = MemoryDB::new();
 		AccountDBMut::from_hash(&mut db, hash).insert(&code[..]);
 
-		let fat_rlp = account::to_fat_rlp(&acc, &AccountDB::from_hash(&db, hash), &mut used_code).unwrap();
+		let fat_rlp = account::to_fat_rlps(&acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value()).unwrap();
 
 		let mut stream = RlpStream::new_list(1);
-		stream.begin_list(2).append(&hash).append_raw(&fat_rlp, 1);
+		stream.begin_list(2).append(&hash).append_raw(&fat_rlp[0], 1);
 		stream.out()
 	};
 
@@ -164,6 +165,7 @@ fn checks_flag() {
 	let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap();
 
 	writer.into_inner().finish(::snapshot::ManifestData {
+		version: 2,
 		state_hashes: state_hashes,
 		block_hashes: Vec::new(),
 		state_root: state_root,

ethcore/src/types/snapshot_manifest.rs

@@ -24,6 +24,8 @@ use util::Bytes;
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[cfg_attr(feature = "ipc", binary)]
 pub struct ManifestData {
+	/// Snapshot format version.
+	pub version: u64,
 	/// List of state chunk hashes.
 	pub state_hashes: Vec<H256>,
 	/// List of block chunk hashes.
@@ -39,7 +41,8 @@ pub struct ManifestData {
 impl ManifestData {
 	/// Encode the manifest data to rlp.
 	pub fn into_rlp(self) -> Bytes {
-		let mut stream = RlpStream::new_list(5);
+		let mut stream = RlpStream::new_list(6);
+		stream.append(&self.version);
 		stream.append(&self.state_hashes);
 		stream.append(&self.block_hashes);
 		stream.append(&self.state_root);
@@ -52,14 +55,20 @@ impl ManifestData {
 	/// Try to restore manifest data from raw bytes, interpreted as RLP.
 	pub fn from_rlp(raw: &[u8]) -> Result<Self, DecoderError> {
 		let decoder = UntrustedRlp::new(raw);
+		let (start, version) = if decoder.item_count() == 5 {
+			(0, 1)
+		} else {
+			(1, decoder.val_at(0)?)
+		};
 
-		let state_hashes: Vec<H256> = decoder.val_at(0)?;
+		let state_hashes: Vec<H256> = decoder.val_at(start + 0)?;
-		let block_hashes: Vec<H256> = decoder.val_at(1)?;
+		let block_hashes: Vec<H256> = decoder.val_at(start + 1)?;
-		let state_root: H256 = decoder.val_at(2)?;
+		let state_root: H256 = decoder.val_at(start + 2)?;
-		let block_number: u64 = decoder.val_at(3)?;
+		let block_number: u64 = decoder.val_at(start + 3)?;
-		let block_hash: H256 = decoder.val_at(4)?;
+		let block_hash: H256 = decoder.val_at(start + 4)?;
 
 		Ok(ManifestData {
+			version: version,
 			state_hashes: state_hashes,
 			block_hashes: block_hashes,
 			state_root: state_root,

mac/Parity.pkgproj

@@ -578,7 +578,7 @@
 				<key>OVERWRITE_PERMISSIONS</key>
 				<false/>
 				<key>VERSION</key>
-				<string>1.5.11</string>
+				<string>1.5.12</string>
 			</dict>
 			<key>UUID</key>
 			<string>2DCD5B81-7BAF-4DA1-9251-6274B089FD36</string>

nsis/installer.nsi

@@ -10,7 +10,7 @@
 !define DESCRIPTION "Fast, light, robust Ethereum implementation"
 !define VERSIONMAJOR 1
 !define VERSIONMINOR 5
-!define VERSIONBUILD 11
+!define VERSIONBUILD 12
 !define ARGS "--warp"
 !define FIRST_START_ARGS "ui --warp --mode=passive"
 

sync/src/chain.rs

@@ -158,6 +158,8 @@ pub const SNAPSHOT_SYNC_PACKET_COUNT: u8 = 0x16;
 
 const MAX_SNAPSHOT_CHUNKS_DOWNLOAD_AHEAD: usize = 3;
 
+const MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION: u64 = 1;
+
 const WAIT_PEERS_TIMEOUT_SEC: u64 = 5;
 const STATUS_TIMEOUT_SEC: u64 = 5;
 const HEADERS_TIMEOUT_SEC: u64 = 15;
@@ -1033,12 +1035,18 @@ impl ChainSync {
 		let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) {
 			Err(e) => {
 				trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e);
-				io.disconnect_peer(peer_id);
+				io.disable_peer(peer_id);
 				self.continue_sync(io);
 				return Ok(());
 			}
 			Ok(manifest) => manifest,
 		};
+		if manifest.version < MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION {
+			trace!(target: "sync", "{}: Snapshot manifest version too low: {}", peer_id, manifest.version);
+			io.disable_peer(peer_id);
+			self.continue_sync(io);
+			return Ok(());
+		}
 		self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3());
 		io.snapshot_service().begin_restore(manifest);
 		self.state = SyncState::SnapshotData;

sync/src/snapshot.rs

@@ -139,6 +139,7 @@ mod test {
 		let state_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
 		let block_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
 		let manifest = ManifestData {
+			version: 2,
 			state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
 			block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
 			state_root: H256::new(),

sync/src/tests/snapshot.rs

@@ -49,6 +49,7 @@ impl TestSnapshotService {
 		let state_chunks: Vec<Bytes> = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect();
 		let block_chunks: Vec<Bytes> = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect();
 		let manifest = ManifestData {
+			version: 2,
 			state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
 			block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
 			state_root: H256::new(),

util/Cargo.toml

@@ -3,7 +3,7 @@ description = "Ethcore utility library"
 homepage = "http://parity.io"
 license = "GPL-3.0"
 name = "ethcore-util"
-version = "1.5.11"
+version = "1.5.12"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"
 

util/src/journaldb/overlayrecentdb.rs

@@ -347,10 +347,7 @@ impl JournalDB for OverlayRecentDB {
 
 			match rc {
 				0 => {}
-				1 => {
+				_ if rc > 0 => {
-					if cfg!(debug_assertions) && self.backing.get(self.column, &key)?.is_some() {
-						return Err(BaseDataError::AlreadyExists(key).into());
-					}
 					batch.put(self.column, &key, &value)
 				}
 				-1 => {
@@ -359,7 +356,7 @@ impl JournalDB for OverlayRecentDB {
 					}
 					batch.delete(self.column, &key)
 				}
-				_ => panic!("Attempted to inject invalid state."),
+				_ => panic!("Attempted to inject invalid state ({})", rc),
 			}
 		}