Snapshot format changes (#2234)

Closes #2213
Omit transaction and receipt roots from abridged block.
No longer use RLP compression.

Make ordered_trie_root generic over an iterator to save an allocation.

Breaks snapshot format backwards compatibility (with other 1.4 snapshots -- it's already been broken with 1.3).

Documentation will need updating

ethcore/src/block.rs

@@ -350,11 +350,11 @@ impl<'x> OpenBlock<'x> {
 		s.block.state.snapshot();
 
 		s.engine.on_close_block(&mut s.block);
-		s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect()));
+		s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
 		let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
 		s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
 		s.block.base.header.set_state_root(s.block.state.root().clone());
-		s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect()));
+		s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
 		s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator
 		s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used));
 
@@ -374,14 +374,14 @@ impl<'x> OpenBlock<'x> {
 
 		s.engine.on_close_block(&mut s.block);
 		if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP {
-			s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect()));
+			s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
 		}
 		let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
 		if s.block.base.header.uncles_hash().is_zero() {
 			s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
 		}
 		if s.block.base.header.receipts_root().is_zero() || s.block.base.header.receipts_root() == &SHA3_NULL_RLP {
-			s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect()));
+			s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
 		}
 
 		s.block.base.header.set_state_root(s.block.state.root().clone());

ethcore/src/snapshot/block.rs

@@ -21,10 +21,10 @@ use header::Header;
 
 use views::BlockView;
 use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View};
-use rlp::{Compressible, RlpType};
 use util::{Bytes, Hashable, H256};
+use util::triehash::ordered_trie_root;
 
-const HEADER_FIELDS: usize = 10;
+const HEADER_FIELDS: usize = 8;
 const BLOCK_FIELDS: usize = 2;
 
 pub struct AbridgedBlock {
@@ -61,8 +61,6 @@ impl AbridgedBlock {
 		stream
 			.append(&header.author())
 			.append(&header.state_root())
-			.append(&header.transactions_root())
-			.append(&header.receipts_root())
 			.append(&header.log_bloom())
 			.append(&header.difficulty())
 			.append(&header.gas_limit())
@@ -79,33 +77,35 @@ impl AbridgedBlock {
 		}
 
 		AbridgedBlock {
-			rlp: UntrustedRlp::new(stream.as_raw()).compress(RlpType::Blocks).to_vec(),
+			rlp: stream.out(),
 		}
 	}
 
 	/// Flesh out an abridged block view with the provided parent hash and block number.
 	///
 	/// Will fail if contains invalid rlp.
-	pub fn to_block(&self, parent_hash: H256, number: u64) -> Result<Block, DecoderError> {
+	pub fn to_block(&self, parent_hash: H256, number: u64, receipts_root: H256) -> Result<Block, DecoderError> {
-		let rlp = UntrustedRlp::new(&self.rlp).decompress(RlpType::Blocks);
+		let rlp = UntrustedRlp::new(&self.rlp);
-		let rlp = UntrustedRlp::new(&rlp);
 
 		let mut header: Header = Default::default();
 		header.set_parent_hash(parent_hash);
 		header.set_author(try!(rlp.val_at(0)));
 		header.set_state_root(try!(rlp.val_at(1)));
-		header.set_transactions_root(try!(rlp.val_at(2)));
+		header.set_log_bloom(try!(rlp.val_at(2)));
-		header.set_receipts_root(try!(rlp.val_at(3)));
+		header.set_difficulty(try!(rlp.val_at(3)));
-		header.set_log_bloom(try!(rlp.val_at(4)));
-		header.set_difficulty(try!(rlp.val_at(5)));
 		header.set_number(number);
-		header.set_gas_limit(try!(rlp.val_at(6)));
+		header.set_gas_limit(try!(rlp.val_at(4)));
-		header.set_gas_used(try!(rlp.val_at(7)));
+		header.set_gas_used(try!(rlp.val_at(5)));
-		header.set_timestamp(try!(rlp.val_at(8)));
+		header.set_timestamp(try!(rlp.val_at(6)));
-		header.set_extra_data(try!(rlp.val_at(9)));
+		header.set_extra_data(try!(rlp.val_at(7)));
 
-		let transactions = try!(rlp.val_at(10));
+		let transactions = try!(rlp.val_at(8));
-		let uncles: Vec<Header> = try!(rlp.val_at(11));
+		let uncles: Vec<Header> = try!(rlp.val_at(9));
+
+		header.set_transactions_root(ordered_trie_root(
+			try!(rlp.at(8)).iter().map(|r| r.as_raw().to_owned())
+		));
+		header.set_receipts_root(receipts_root);
 
 		let mut uncles_rlp = RlpStream::new();
 		uncles_rlp.append(&uncles);
@@ -143,20 +143,22 @@ mod tests {
 	#[test]
 	fn empty_block_abridging() {
 		let b = Block::default();
+		let receipts_root = b.header.receipts_root().clone();
 		let encoded = encode_block(&b);
 
 		let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
-		assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b);
+		assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
 	}
 
 	#[test]
 	#[should_panic]
 	fn wrong_number() {
 		let b = Block::default();
+		let receipts_root = b.header.receipts_root().clone();
 		let encoded = encode_block(&b);
 
 		let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
-		assert_eq!(abridged.to_block(H256::new(), 2).unwrap(), b);
+		assert_eq!(abridged.to_block(H256::new(), 2, receipts_root).unwrap(), b);
 	}
 
 	#[test]
@@ -184,9 +186,14 @@ mod tests {
 		b.transactions.push(t1);
 		b.transactions.push(t2);
 
+		let receipts_root = b.header.receipts_root().clone();
+		b.header.set_transactions_root(::util::triehash::ordered_trie_root(
+			b.transactions.iter().map(::rlp::encode).map(|out| out.to_vec())
+		));
+
 		let encoded = encode_block(&b);
 
 		let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded[..]));
-		assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b);
+		assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
 	}
 }

ethcore/src/snapshot/mod.rs

@@ -15,6 +15,9 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Snapshot creation, restoration, and network service.
+//!
+//! Documentation of the format can be found at
+//! https://github.com/ethcore/parity/wiki/%22PV64%22-Snapshot-Format
 
 use std::collections::{HashMap, HashSet, VecDeque};
 use std::sync::Arc;
@@ -34,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB};
 use util::kvdb::Database;
 use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
 use util::sha3::SHA3_NULL_RLP;
-use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
+use rlp::{RlpStream, Stream, UntrustedRlp, View};
 
 use self::account::Account;
 use self::block::AbridgedBlock;
@@ -366,8 +369,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
 		let account_db = AccountDB::from_hash(db, account_key_hash);
 
 		let fat_rlp = try!(account.to_fat_rlp(&account_db, &mut used_code));
-		let compressed_rlp = UntrustedRlp::new(&fat_rlp).compress(RlpType::Snapshot).to_vec();
+		try!(chunker.push(account_key, fat_rlp));
-		try!(chunker.push(account_key, compressed_rlp));
 	}
 
 	if chunker.cur_size != 0 {
@@ -508,8 +510,7 @@ fn rebuild_accounts(
 		let account_rlp = UntrustedRlp::new(account_pair);
 
 		let hash: H256 = try!(account_rlp.val_at(0));
-		let decompressed = try!(account_rlp.at(1)).decompress(RlpType::Snapshot);
+		let fat_rlp = try!(account_rlp.at(1));
-		let fat_rlp = UntrustedRlp::new(&decompressed[..]);
 
 		let thin_rlp = {
 			let mut acct_db = AccountDBMut::from_hash(db, hash);
@@ -570,6 +571,7 @@ impl BlockRebuilder {
 	pub fn feed(&mut self, chunk: &[u8], engine: &Engine) -> Result<u64, ::error::Error> {
 		use basic_types::Seal::With;
 		use util::U256;
+		use util::triehash::ordered_trie_root;
 
 		let rlp = UntrustedRlp::new(chunk);
 		let item_count = rlp.item_count();
@@ -586,7 +588,11 @@ impl BlockRebuilder {
 			let abridged_rlp = try!(pair.at(0)).as_raw().to_owned();
 			let abridged_block = AbridgedBlock::from_raw(abridged_rlp);
 			let receipts: Vec<::receipt::Receipt> = try!(pair.val_at(1));
-			let block = try!(abridged_block.to_block(parent_hash, cur_number));
+			let receipts_root = ordered_trie_root(
+				try!(pair.at(1)).iter().map(|r| r.as_raw().to_owned())
+			);
+
+			let block = try!(abridged_block.to_block(parent_hash, cur_number, receipts_root));
 			let block_bytes = block.rlp_bytes(With);
 
 			if self.rng.gen::<f32>() <= POW_VERIFY_RATE {

ethcore/src/verification/verification.rs

@@ -215,7 +215,7 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
 fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> {
 	let block = UntrustedRlp::new(block);
 	let tx = try!(block.at(1));
-	let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here
+	let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
 	if expected_root != transactions_root {
 		return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() })))
 	}
@@ -422,7 +422,7 @@ mod tests {
 		let mut uncles_rlp = RlpStream::new();
 		uncles_rlp.append(&good_uncles);
 		let good_uncles_hash = uncles_rlp.as_raw().sha3();
-		let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()).collect());
+		let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()));
 
 		let mut parent = good.clone();
 		parent.set_number(9);

sync/src/blocks.rs

@@ -233,7 +233,7 @@ impl BlockCollection {
 	fn insert_body(&mut self, b: Bytes) -> Result<(), NetworkError> {
 		let body = UntrustedRlp::new(&b);
 		let tx = try!(body.at(0));
-		let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here
+		let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
 		let uncles = try!(body.at(1)).as_raw().sha3();
 		let header_id = HeaderId {
 			transactions_root: tx_root,

util/src/triehash.rs

@@ -40,7 +40,9 @@ use vector::SharedPrefix;
 /// 	assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap());
 /// }
 /// ```
-pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
+pub fn ordered_trie_root<I>(input: I) -> H256
+	where I: IntoIterator<Item=Vec<u8>>
+{
 	let gen_input = input
 		// first put elements into btree to sort them by nibbles
 		// optimize it later