Snapshot format changes (#2234)

Closes #2213
Omit transaction and receipt roots from abridged block.
No longer use RLP compression.

Make ordered_trie_root generic over an iterator to save an allocation.

Breaks snapshot format backwards compatibility (with other 1.4 snapshots -- it's already been broken with 1.3).

Documentation will need updating
This commit is contained in:
Robert Habermeier 2016-09-22 19:47:03 +02:00 committed by Gav Wood
parent 862feb7172
commit 723d837d05
6 changed files with 50 additions and 35 deletions

View File

@ -350,11 +350,11 @@ impl<'x> OpenBlock<'x> {
s.block.state.snapshot(); s.block.state.snapshot();
s.engine.on_close_block(&mut s.block); s.engine.on_close_block(&mut s.block);
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect())); s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out(); let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
s.block.base.header.set_uncles_hash(uncle_bytes.sha3()); s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
s.block.base.header.set_state_root(s.block.state.root().clone()); s.block.base.header.set_state_root(s.block.state.root().clone());
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect())); s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator
s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used)); s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used));
@ -374,14 +374,14 @@ impl<'x> OpenBlock<'x> {
s.engine.on_close_block(&mut s.block); s.engine.on_close_block(&mut s.block);
if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP { if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP {
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect())); s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
} }
let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out(); let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
if s.block.base.header.uncles_hash().is_zero() { if s.block.base.header.uncles_hash().is_zero() {
s.block.base.header.set_uncles_hash(uncle_bytes.sha3()); s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
} }
if s.block.base.header.receipts_root().is_zero() || s.block.base.header.receipts_root() == &SHA3_NULL_RLP { if s.block.base.header.receipts_root().is_zero() || s.block.base.header.receipts_root() == &SHA3_NULL_RLP {
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect())); s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
} }
s.block.base.header.set_state_root(s.block.state.root().clone()); s.block.base.header.set_state_root(s.block.state.root().clone());

View File

@ -21,10 +21,10 @@ use header::Header;
use views::BlockView; use views::BlockView;
use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View}; use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View};
use rlp::{Compressible, RlpType};
use util::{Bytes, Hashable, H256}; use util::{Bytes, Hashable, H256};
use util::triehash::ordered_trie_root;
const HEADER_FIELDS: usize = 10; const HEADER_FIELDS: usize = 8;
const BLOCK_FIELDS: usize = 2; const BLOCK_FIELDS: usize = 2;
pub struct AbridgedBlock { pub struct AbridgedBlock {
@ -61,8 +61,6 @@ impl AbridgedBlock {
stream stream
.append(&header.author()) .append(&header.author())
.append(&header.state_root()) .append(&header.state_root())
.append(&header.transactions_root())
.append(&header.receipts_root())
.append(&header.log_bloom()) .append(&header.log_bloom())
.append(&header.difficulty()) .append(&header.difficulty())
.append(&header.gas_limit()) .append(&header.gas_limit())
@ -79,33 +77,35 @@ impl AbridgedBlock {
} }
AbridgedBlock { AbridgedBlock {
rlp: UntrustedRlp::new(stream.as_raw()).compress(RlpType::Blocks).to_vec(), rlp: stream.out(),
} }
} }
/// Flesh out an abridged block view with the provided parent hash and block number. /// Flesh out an abridged block view with the provided parent hash and block number.
/// ///
/// Will fail if contains invalid rlp. /// Will fail if contains invalid rlp.
pub fn to_block(&self, parent_hash: H256, number: u64) -> Result<Block, DecoderError> { pub fn to_block(&self, parent_hash: H256, number: u64, receipts_root: H256) -> Result<Block, DecoderError> {
let rlp = UntrustedRlp::new(&self.rlp).decompress(RlpType::Blocks); let rlp = UntrustedRlp::new(&self.rlp);
let rlp = UntrustedRlp::new(&rlp);
let mut header: Header = Default::default(); let mut header: Header = Default::default();
header.set_parent_hash(parent_hash); header.set_parent_hash(parent_hash);
header.set_author(try!(rlp.val_at(0))); header.set_author(try!(rlp.val_at(0)));
header.set_state_root(try!(rlp.val_at(1))); header.set_state_root(try!(rlp.val_at(1)));
header.set_transactions_root(try!(rlp.val_at(2))); header.set_log_bloom(try!(rlp.val_at(2)));
header.set_receipts_root(try!(rlp.val_at(3))); header.set_difficulty(try!(rlp.val_at(3)));
header.set_log_bloom(try!(rlp.val_at(4)));
header.set_difficulty(try!(rlp.val_at(5)));
header.set_number(number); header.set_number(number);
header.set_gas_limit(try!(rlp.val_at(6))); header.set_gas_limit(try!(rlp.val_at(4)));
header.set_gas_used(try!(rlp.val_at(7))); header.set_gas_used(try!(rlp.val_at(5)));
header.set_timestamp(try!(rlp.val_at(8))); header.set_timestamp(try!(rlp.val_at(6)));
header.set_extra_data(try!(rlp.val_at(9))); header.set_extra_data(try!(rlp.val_at(7)));
let transactions = try!(rlp.val_at(10)); let transactions = try!(rlp.val_at(8));
let uncles: Vec<Header> = try!(rlp.val_at(11)); let uncles: Vec<Header> = try!(rlp.val_at(9));
header.set_transactions_root(ordered_trie_root(
try!(rlp.at(8)).iter().map(|r| r.as_raw().to_owned())
));
header.set_receipts_root(receipts_root);
let mut uncles_rlp = RlpStream::new(); let mut uncles_rlp = RlpStream::new();
uncles_rlp.append(&uncles); uncles_rlp.append(&uncles);
@ -143,20 +143,22 @@ mod tests {
#[test] #[test]
fn empty_block_abridging() { fn empty_block_abridging() {
let b = Block::default(); let b = Block::default();
let receipts_root = b.header.receipts_root().clone();
let encoded = encode_block(&b); let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded)); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b); assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
} }
#[test] #[test]
#[should_panic] #[should_panic]
fn wrong_number() { fn wrong_number() {
let b = Block::default(); let b = Block::default();
let receipts_root = b.header.receipts_root().clone();
let encoded = encode_block(&b); let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded)); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
assert_eq!(abridged.to_block(H256::new(), 2).unwrap(), b); assert_eq!(abridged.to_block(H256::new(), 2, receipts_root).unwrap(), b);
} }
#[test] #[test]
@ -184,9 +186,14 @@ mod tests {
b.transactions.push(t1); b.transactions.push(t1);
b.transactions.push(t2); b.transactions.push(t2);
let receipts_root = b.header.receipts_root().clone();
b.header.set_transactions_root(::util::triehash::ordered_trie_root(
b.transactions.iter().map(::rlp::encode).map(|out| out.to_vec())
));
let encoded = encode_block(&b); let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded[..])); let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded[..]));
assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b); assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
} }
} }

View File

@ -15,6 +15,9 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>. // along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Snapshot creation, restoration, and network service. //! Snapshot creation, restoration, and network service.
//!
//! Documentation of the format can be found at
//! https://github.com/ethcore/parity/wiki/%22PV64%22-Snapshot-Format
use std::collections::{HashMap, HashSet, VecDeque}; use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc; use std::sync::Arc;
@ -34,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB};
use util::kvdb::Database; use util::kvdb::Database;
use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut}; use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
use util::sha3::SHA3_NULL_RLP; use util::sha3::SHA3_NULL_RLP;
use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType}; use rlp::{RlpStream, Stream, UntrustedRlp, View};
use self::account::Account; use self::account::Account;
use self::block::AbridgedBlock; use self::block::AbridgedBlock;
@ -366,8 +369,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
let account_db = AccountDB::from_hash(db, account_key_hash); let account_db = AccountDB::from_hash(db, account_key_hash);
let fat_rlp = try!(account.to_fat_rlp(&account_db, &mut used_code)); let fat_rlp = try!(account.to_fat_rlp(&account_db, &mut used_code));
let compressed_rlp = UntrustedRlp::new(&fat_rlp).compress(RlpType::Snapshot).to_vec(); try!(chunker.push(account_key, fat_rlp));
try!(chunker.push(account_key, compressed_rlp));
} }
if chunker.cur_size != 0 { if chunker.cur_size != 0 {
@ -508,8 +510,7 @@ fn rebuild_accounts(
let account_rlp = UntrustedRlp::new(account_pair); let account_rlp = UntrustedRlp::new(account_pair);
let hash: H256 = try!(account_rlp.val_at(0)); let hash: H256 = try!(account_rlp.val_at(0));
let decompressed = try!(account_rlp.at(1)).decompress(RlpType::Snapshot); let fat_rlp = try!(account_rlp.at(1));
let fat_rlp = UntrustedRlp::new(&decompressed[..]);
let thin_rlp = { let thin_rlp = {
let mut acct_db = AccountDBMut::from_hash(db, hash); let mut acct_db = AccountDBMut::from_hash(db, hash);
@ -570,6 +571,7 @@ impl BlockRebuilder {
pub fn feed(&mut self, chunk: &[u8], engine: &Engine) -> Result<u64, ::error::Error> { pub fn feed(&mut self, chunk: &[u8], engine: &Engine) -> Result<u64, ::error::Error> {
use basic_types::Seal::With; use basic_types::Seal::With;
use util::U256; use util::U256;
use util::triehash::ordered_trie_root;
let rlp = UntrustedRlp::new(chunk); let rlp = UntrustedRlp::new(chunk);
let item_count = rlp.item_count(); let item_count = rlp.item_count();
@ -586,7 +588,11 @@ impl BlockRebuilder {
let abridged_rlp = try!(pair.at(0)).as_raw().to_owned(); let abridged_rlp = try!(pair.at(0)).as_raw().to_owned();
let abridged_block = AbridgedBlock::from_raw(abridged_rlp); let abridged_block = AbridgedBlock::from_raw(abridged_rlp);
let receipts: Vec<::receipt::Receipt> = try!(pair.val_at(1)); let receipts: Vec<::receipt::Receipt> = try!(pair.val_at(1));
let block = try!(abridged_block.to_block(parent_hash, cur_number)); let receipts_root = ordered_trie_root(
try!(pair.at(1)).iter().map(|r| r.as_raw().to_owned())
);
let block = try!(abridged_block.to_block(parent_hash, cur_number, receipts_root));
let block_bytes = block.rlp_bytes(With); let block_bytes = block.rlp_bytes(With);
if self.rng.gen::<f32>() <= POW_VERIFY_RATE { if self.rng.gen::<f32>() <= POW_VERIFY_RATE {

View File

@ -215,7 +215,7 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> { fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> {
let block = UntrustedRlp::new(block); let block = UntrustedRlp::new(block);
let tx = try!(block.at(1)); let tx = try!(block.at(1));
let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
if expected_root != transactions_root { if expected_root != transactions_root {
return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() }))) return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() })))
} }
@ -422,7 +422,7 @@ mod tests {
let mut uncles_rlp = RlpStream::new(); let mut uncles_rlp = RlpStream::new();
uncles_rlp.append(&good_uncles); uncles_rlp.append(&good_uncles);
let good_uncles_hash = uncles_rlp.as_raw().sha3(); let good_uncles_hash = uncles_rlp.as_raw().sha3();
let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()).collect()); let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()));
let mut parent = good.clone(); let mut parent = good.clone();
parent.set_number(9); parent.set_number(9);

View File

@ -233,7 +233,7 @@ impl BlockCollection {
fn insert_body(&mut self, b: Bytes) -> Result<(), NetworkError> { fn insert_body(&mut self, b: Bytes) -> Result<(), NetworkError> {
let body = UntrustedRlp::new(&b); let body = UntrustedRlp::new(&b);
let tx = try!(body.at(0)); let tx = try!(body.at(0));
let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
let uncles = try!(body.at(1)).as_raw().sha3(); let uncles = try!(body.at(1)).as_raw().sha3();
let header_id = HeaderId { let header_id = HeaderId {
transactions_root: tx_root, transactions_root: tx_root,

View File

@ -40,7 +40,9 @@ use vector::SharedPrefix;
/// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap()); /// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap());
/// } /// }
/// ``` /// ```
pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 { pub fn ordered_trie_root<I>(input: I) -> H256
where I: IntoIterator<Item=Vec<u8>>
{
let gen_input = input let gen_input = input
// first put elements into btree to sort them by nibbles // first put elements into btree to sort them by nibbles
// optimize it later // optimize it later