From 723d837d0542022e9f566c16d095a49517be6070 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 22 Sep 2016 19:47:03 +0200 Subject: [PATCH] Snapshot format changes (#2234) Closes #2213 Omit transaction and receipt roots from abridged block. No longer use RLP compression. Make ordered_trie_root generic over an iterator to save an allocation. Breaks snapshot format backwards compatibility (with other 1.4 snapshots -- it's already been broken with 1.3). Documentation will need updating --- ethcore/src/block.rs | 8 ++-- ethcore/src/snapshot/block.rs | 49 ++++++++++++++---------- ethcore/src/snapshot/mod.rs | 18 ++++++--- ethcore/src/verification/verification.rs | 4 +- sync/src/blocks.rs | 2 +- util/src/triehash.rs | 4 +- 6 files changed, 50 insertions(+), 35 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 87f4749a1..b35b4dc1a 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -350,11 +350,11 @@ impl<'x> OpenBlock<'x> { s.block.state.snapshot(); s.engine.on_close_block(&mut s.block); - s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect())); + s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()))); let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out(); s.block.base.header.set_uncles_hash(uncle_bytes.sha3()); s.block.base.header.set_state_root(s.block.state.root().clone()); - s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect())); + s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()))); s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used)); @@ -374,14 +374,14 @@ impl<'x> OpenBlock<'x> { s.engine.on_close_block(&mut s.block); if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP { - s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect())); + s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()))); } let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out(); if s.block.base.header.uncles_hash().is_zero() { s.block.base.header.set_uncles_hash(uncle_bytes.sha3()); } if s.block.base.header.receipts_root().is_zero() || s.block.base.header.receipts_root() == &SHA3_NULL_RLP { - s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect())); + s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()))); } s.block.base.header.set_state_root(s.block.state.root().clone()); diff --git a/ethcore/src/snapshot/block.rs b/ethcore/src/snapshot/block.rs index 05b3281c8..4f7f912ca 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/src/snapshot/block.rs @@ -21,10 +21,10 @@ use header::Header; use views::BlockView; use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View}; -use rlp::{Compressible, RlpType}; use util::{Bytes, Hashable, H256}; +use util::triehash::ordered_trie_root; -const HEADER_FIELDS: usize = 10; +const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; pub struct AbridgedBlock { @@ -61,8 +61,6 @@ impl AbridgedBlock { stream .append(&header.author()) .append(&header.state_root()) - .append(&header.transactions_root()) - .append(&header.receipts_root()) .append(&header.log_bloom()) .append(&header.difficulty()) .append(&header.gas_limit()) @@ -79,33 +77,35 @@ impl AbridgedBlock { } AbridgedBlock { - rlp: UntrustedRlp::new(stream.as_raw()).compress(RlpType::Blocks).to_vec(), + rlp: stream.out(), } } /// Flesh out an abridged block view with the provided parent hash and block number. /// /// Will fail if contains invalid rlp. - pub fn to_block(&self, parent_hash: H256, number: u64) -> Result { - let rlp = UntrustedRlp::new(&self.rlp).decompress(RlpType::Blocks); - let rlp = UntrustedRlp::new(&rlp); + pub fn to_block(&self, parent_hash: H256, number: u64, receipts_root: H256) -> Result { + let rlp = UntrustedRlp::new(&self.rlp); let mut header: Header = Default::default(); header.set_parent_hash(parent_hash); header.set_author(try!(rlp.val_at(0))); header.set_state_root(try!(rlp.val_at(1))); - header.set_transactions_root(try!(rlp.val_at(2))); - header.set_receipts_root(try!(rlp.val_at(3))); - header.set_log_bloom(try!(rlp.val_at(4))); - header.set_difficulty(try!(rlp.val_at(5))); + header.set_log_bloom(try!(rlp.val_at(2))); + header.set_difficulty(try!(rlp.val_at(3))); header.set_number(number); - header.set_gas_limit(try!(rlp.val_at(6))); - header.set_gas_used(try!(rlp.val_at(7))); - header.set_timestamp(try!(rlp.val_at(8))); - header.set_extra_data(try!(rlp.val_at(9))); + header.set_gas_limit(try!(rlp.val_at(4))); + header.set_gas_used(try!(rlp.val_at(5))); + header.set_timestamp(try!(rlp.val_at(6))); + header.set_extra_data(try!(rlp.val_at(7))); - let transactions = try!(rlp.val_at(10)); - let uncles: Vec
= try!(rlp.val_at(11)); + let transactions = try!(rlp.val_at(8)); + let uncles: Vec
= try!(rlp.val_at(9)); + + header.set_transactions_root(ordered_trie_root( + try!(rlp.at(8)).iter().map(|r| r.as_raw().to_owned()) + )); + header.set_receipts_root(receipts_root); let mut uncles_rlp = RlpStream::new(); uncles_rlp.append(&uncles); @@ -143,20 +143,22 @@ mod tests { #[test] fn empty_block_abridging() { let b = Block::default(); + let receipts_root = b.header.receipts_root().clone(); let encoded = encode_block(&b); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded)); - assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b); + assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b); } #[test] #[should_panic] fn wrong_number() { let b = Block::default(); + let receipts_root = b.header.receipts_root().clone(); let encoded = encode_block(&b); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded)); - assert_eq!(abridged.to_block(H256::new(), 2).unwrap(), b); + assert_eq!(abridged.to_block(H256::new(), 2, receipts_root).unwrap(), b); } #[test] @@ -184,9 +186,14 @@ mod tests { b.transactions.push(t1); b.transactions.push(t2); + let receipts_root = b.header.receipts_root().clone(); + b.header.set_transactions_root(::util::triehash::ordered_trie_root( + b.transactions.iter().map(::rlp::encode).map(|out| out.to_vec()) + )); + let encoded = encode_block(&b); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded[..])); - assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b); + assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b); } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 5e8f3bd7a..2074f8174 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -15,6 +15,9 @@ // along with Parity. If not, see . //! Snapshot creation, restoration, and network service. +//! +//! Documentation of the format can be found at +//! https://github.com/ethcore/parity/wiki/%22PV64%22-Snapshot-Format use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; @@ -34,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB}; use util::kvdb::Database; use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut}; use util::sha3::SHA3_NULL_RLP; -use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType}; +use rlp::{RlpStream, Stream, UntrustedRlp, View}; use self::account::Account; use self::block::AbridgedBlock; @@ -366,8 +369,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex Result { use basic_types::Seal::With; use util::U256; + use util::triehash::ordered_trie_root; let rlp = UntrustedRlp::new(chunk); let item_count = rlp.item_count(); @@ -586,7 +588,11 @@ impl BlockRebuilder { let abridged_rlp = try!(pair.at(0)).as_raw().to_owned(); let abridged_block = AbridgedBlock::from_raw(abridged_rlp); let receipts: Vec<::receipt::Receipt> = try!(pair.val_at(1)); - let block = try!(abridged_block.to_block(parent_hash, cur_number)); + let receipts_root = ordered_trie_root( + try!(pair.at(1)).iter().map(|r| r.as_raw().to_owned()) + ); + + let block = try!(abridged_block.to_block(parent_hash, cur_number, receipts_root)); let block_bytes = block.rlp_bytes(With); if self.rng.gen::() <= POW_VERIFY_RATE { diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 86d3623c2..4e1305a33 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -215,7 +215,7 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> { fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> { let block = UntrustedRlp::new(block); let tx = try!(block.at(1)); - let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here + let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here if expected_root != transactions_root { return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() }))) } @@ -422,7 +422,7 @@ mod tests { let mut uncles_rlp = RlpStream::new(); uncles_rlp.append(&good_uncles); let good_uncles_hash = uncles_rlp.as_raw().sha3(); - let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::(t).to_vec()).collect()); + let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::(t).to_vec())); let mut parent = good.clone(); parent.set_number(9); diff --git a/sync/src/blocks.rs b/sync/src/blocks.rs index 753ba7111..ad842ced6 100644 --- a/sync/src/blocks.rs +++ b/sync/src/blocks.rs @@ -233,7 +233,7 @@ impl BlockCollection { fn insert_body(&mut self, b: Bytes) -> Result<(), NetworkError> { let body = UntrustedRlp::new(&b); let tx = try!(body.at(0)); - let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here + let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here let uncles = try!(body.at(1)).as_raw().sha3(); let header_id = HeaderId { transactions_root: tx_root, diff --git a/util/src/triehash.rs b/util/src/triehash.rs index f49b588d4..c8ab5bb08 100644 --- a/util/src/triehash.rs +++ b/util/src/triehash.rs @@ -40,7 +40,9 @@ use vector::SharedPrefix; /// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap()); /// } /// ``` -pub fn ordered_trie_root(input: Vec>) -> H256 { +pub fn ordered_trie_root(input: I) -> H256 + where I: IntoIterator> +{ let gen_input = input // first put elements into btree to sort them by nibbles // optimize it later