grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Snapshot format changes (#2234)

Browse Source 
Closes #2213
Omit transaction and receipt roots from abridged block.
No longer use RLP compression.

Make ordered_trie_root generic over an iterator to save an allocation.

Breaks snapshot format backwards compatibility (with other 1.4 snapshots -- it's already been broken with 1.3).

Documentation will need updating
This commit is contained in:
Robert Habermeier 2016-09-22 19:47:03 +02:00 committed by Gav Wood
parent 862feb7172
commit 723d837d05
6 changed files with 50 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ethcore/src/block.rs
View File

@ -350,11 +350,11 @@ impl<'x> OpenBlock<'x> {
s.block.state.snapshot();
s.engine.on_close_block(&mut s.block);
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect()));
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
s.block.base.header.set_state_root(s.block.state.root().clone());
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect()));
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator
s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used));
@ -374,14 +374,14 @@ impl<'x> OpenBlock<'x> {
s.engine.on_close_block(&mut s.block);
if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP {
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()).collect()));
s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec())));
}
let uncle_bytes = s.block.base.uncles.iter().fold(RlpStream::new_list(s.block.base.uncles.len()), |mut s, u| {s.append_raw(&u.rlp(Seal::With), 1); s} ).out();
if s.block.base.header.uncles_hash().is_zero() {
s.block.base.header.set_uncles_hash(uncle_bytes.sha3());
}
if s.block.base.header.receipts_root().is_zero() || s.block.base.header.receipts_root() == &SHA3_NULL_RLP {
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec()).collect()));
s.block.base.header.set_receipts_root(ordered_trie_root(s.block.receipts.iter().map(|r| r.rlp_bytes().to_vec())));
}
s.block.base.header.set_state_root(s.block.state.root().clone());

49
ethcore/src/snapshot/block.rs
View File

@ -21,10 +21,10 @@ use header::Header;
use views::BlockView;
use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View};
use rlp::{Compressible, RlpType};
use util::{Bytes, Hashable, H256};
use util::triehash::ordered_trie_root;
const HEADER_FIELDS: usize = 10;
const HEADER_FIELDS: usize = 8;
const BLOCK_FIELDS: usize = 2;
pub struct AbridgedBlock {
@ -61,8 +61,6 @@ impl AbridgedBlock {
stream
.append(&header.author())
.append(&header.state_root())
.append(&header.transactions_root())
.append(&header.receipts_root())
.append(&header.log_bloom())
.append(&header.difficulty())
.append(&header.gas_limit())
@ -79,33 +77,35 @@ impl AbridgedBlock {
}
AbridgedBlock {
rlp: UntrustedRlp::new(stream.as_raw()).compress(RlpType::Blocks).to_vec(),
rlp: stream.out(),
}
}
/// Flesh out an abridged block view with the provided parent hash and block number.
///
/// Will fail if contains invalid rlp.
pub fn to_block(&self, parent_hash: H256, number: u64) -> Result<Block, DecoderError> {
let rlp = UntrustedRlp::new(&self.rlp).decompress(RlpType::Blocks);
let rlp = UntrustedRlp::new(&rlp);
pub fn to_block(&self, parent_hash: H256, number: u64, receipts_root: H256) -> Result<Block, DecoderError> {
let rlp = UntrustedRlp::new(&self.rlp);
let mut header: Header = Default::default();
header.set_parent_hash(parent_hash);
header.set_author(try!(rlp.val_at(0)));
header.set_state_root(try!(rlp.val_at(1)));
header.set_transactions_root(try!(rlp.val_at(2)));
header.set_receipts_root(try!(rlp.val_at(3)));
header.set_log_bloom(try!(rlp.val_at(4)));
header.set_difficulty(try!(rlp.val_at(5)));
header.set_log_bloom(try!(rlp.val_at(2)));
header.set_difficulty(try!(rlp.val_at(3)));
header.set_number(number);
header.set_gas_limit(try!(rlp.val_at(6)));
header.set_gas_used(try!(rlp.val_at(7)));
header.set_timestamp(try!(rlp.val_at(8)));
header.set_extra_data(try!(rlp.val_at(9)));
header.set_gas_limit(try!(rlp.val_at(4)));
header.set_gas_used(try!(rlp.val_at(5)));
header.set_timestamp(try!(rlp.val_at(6)));
header.set_extra_data(try!(rlp.val_at(7)));
let transactions = try!(rlp.val_at(10));
let uncles: Vec<Header> = try!(rlp.val_at(11));
let transactions = try!(rlp.val_at(8));
let uncles: Vec<Header> = try!(rlp.val_at(9));
header.set_transactions_root(ordered_trie_root(
try!(rlp.at(8)).iter().map(|r| r.as_raw().to_owned())
));
header.set_receipts_root(receipts_root);
let mut uncles_rlp = RlpStream::new();
uncles_rlp.append(&uncles);
@ -143,20 +143,22 @@ mod tests {
#[test]
fn empty_block_abridging() {
let b = Block::default();
let receipts_root = b.header.receipts_root().clone();
let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b);
assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
}
#[test]
#[should_panic]
fn wrong_number() {
let b = Block::default();
let receipts_root = b.header.receipts_root().clone();
let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded));
assert_eq!(abridged.to_block(H256::new(), 2).unwrap(), b);
assert_eq!(abridged.to_block(H256::new(), 2, receipts_root).unwrap(), b);
}
#[test]
@ -184,9 +186,14 @@ mod tests {
b.transactions.push(t1);
b.transactions.push(t2);
let receipts_root = b.header.receipts_root().clone();
b.header.set_transactions_root(::util::triehash::ordered_trie_root(
b.transactions.iter().map(::rlp::encode).map(|out| out.to_vec())
));
let encoded = encode_block(&b);
let abridged = AbridgedBlock::from_block_view(&BlockView::new(&encoded[..]));
assert_eq!(abridged.to_block(H256::new(), 0).unwrap(), b);
assert_eq!(abridged.to_block(H256::new(), 0, receipts_root).unwrap(), b);
}
}

18
ethcore/src/snapshot/mod.rs
View File

@ -15,6 +15,9 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Snapshot creation, restoration, and network service.
//!
//! Documentation of the format can be found at
//! https://github.com/ethcore/parity/wiki/%22PV64%22-Snapshot-Format
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
@ -34,7 +37,7 @@ use util::journaldb::{self, Algorithm, JournalDB};
use util::kvdb::Database;
use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
use util::sha3::SHA3_NULL_RLP;
use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
use rlp::{RlpStream, Stream, UntrustedRlp, View};
use self::account::Account;
use self::block::AbridgedBlock;
@ -366,8 +369,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
let account_db = AccountDB::from_hash(db, account_key_hash);
let fat_rlp = try!(account.to_fat_rlp(&account_db, &mut used_code));
let compressed_rlp = UntrustedRlp::new(&fat_rlp).compress(RlpType::Snapshot).to_vec();
try!(chunker.push(account_key, compressed_rlp));
try!(chunker.push(account_key, fat_rlp));
}
if chunker.cur_size != 0 {
@ -508,8 +510,7 @@ fn rebuild_accounts(
let account_rlp = UntrustedRlp::new(account_pair);
let hash: H256 = try!(account_rlp.val_at(0));
let decompressed = try!(account_rlp.at(1)).decompress(RlpType::Snapshot);
let fat_rlp = UntrustedRlp::new(&decompressed[..]);
let fat_rlp = try!(account_rlp.at(1));
let thin_rlp = {
let mut acct_db = AccountDBMut::from_hash(db, hash);
@ -570,6 +571,7 @@ impl BlockRebuilder {
pub fn feed(&mut self, chunk: &[u8], engine: &Engine) -> Result<u64, ::error::Error> {
use basic_types::Seal::With;
use util::U256;
use util::triehash::ordered_trie_root;
let rlp = UntrustedRlp::new(chunk);
let item_count = rlp.item_count();
@ -586,7 +588,11 @@ impl BlockRebuilder {
let abridged_rlp = try!(pair.at(0)).as_raw().to_owned();
let abridged_block = AbridgedBlock::from_raw(abridged_rlp);
let receipts: Vec<::receipt::Receipt> = try!(pair.val_at(1));
let block = try!(abridged_block.to_block(parent_hash, cur_number));
let receipts_root = ordered_trie_root(
try!(pair.at(1)).iter().map(|r| r.as_raw().to_owned())
);
let block = try!(abridged_block.to_block(parent_hash, cur_number, receipts_root));
let block_bytes = block.rlp_bytes(With);
if self.rng.gen::<f32>() <= POW_VERIFY_RATE {

4
ethcore/src/verification/verification.rs
View File

@ -215,7 +215,7 @@ fn verify_parent(header: &Header, parent: &Header) -> Result<(), Error> {
fn verify_block_integrity(block: &[u8], transactions_root: &H256, uncles_hash: &H256) -> Result<(), Error> {
let block = UntrustedRlp::new(block);
let tx = try!(block.at(1));
let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here
let expected_root = &ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
if expected_root != transactions_root {
return Err(From::from(BlockError::InvalidTransactionsRoot(Mismatch { expected: expected_root.clone(), found: transactions_root.clone() })))
}
@ -422,7 +422,7 @@ mod tests {
let mut uncles_rlp = RlpStream::new();
uncles_rlp.append(&good_uncles);
let good_uncles_hash = uncles_rlp.as_raw().sha3();
let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()).collect());
let good_transactions_root = ordered_trie_root(good_transactions.iter().map(|t| ::rlp::encode::<SignedTransaction>(t).to_vec()));
let mut parent = good.clone();
parent.set_number(9);

2
sync/src/blocks.rs
View File

@ -233,7 +233,7 @@ impl BlockCollection {
fn insert_body(&mut self, b: Bytes) -> Result<(), NetworkError> {
let body = UntrustedRlp::new(&b);
let tx = try!(body.at(0));
let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec()).collect()); //TODO: get rid of vectors here
let tx_root = ordered_trie_root(tx.iter().map(|r| r.as_raw().to_vec())); //TODO: get rid of vectors here
let uncles = try!(body.at(1)).as_raw().sha3();
let header_id = HeaderId {
transactions_root: tx_root,

4
util/src/triehash.rs
View File

@ -40,7 +40,9 @@ use vector::SharedPrefix;
/// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap());
/// }
/// ```
pub fn ordered_trie_root(input: Vec<Vec<u8>>) -> H256 {
pub fn ordered_trie_root<I>(input: I) -> H256
where I: IntoIterator<Item=Vec<u8>>
{
let gen_input = input
// first put elements into btree to sort them by nibbles
// optimize it later