From 0caaa019651aa0746f48978a9caea92b2a95775d Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Mon, 27 Mar 2017 19:11:24 +0200 Subject: [PATCH] [stable] Backports (#5297) * v1.5.12 * Fine grained snapshot chunking --- Cargo.lock | 66 +++++++----- Cargo.toml | 2 +- ethcore/Cargo.toml | 1 + ethcore/src/lib.rs | 1 + ethcore/src/snapshot/account.rs | 143 +++++++++++++++---------- ethcore/src/snapshot/error.rs | 3 + ethcore/src/snapshot/io.rs | 34 ++++-- ethcore/src/snapshot/mod.rs | 30 ++++-- ethcore/src/snapshot/service.rs | 1 + ethcore/src/snapshot/tests/blocks.rs | 2 + ethcore/src/snapshot/tests/mod.rs | 3 +- ethcore/src/snapshot/tests/service.rs | 1 + ethcore/src/snapshot/tests/state.rs | 6 +- ethcore/src/types/snapshot_manifest.rs | 21 ++-- mac/Parity.pkgproj | 2 +- nsis/installer.nsi | 2 +- sync/src/chain.rs | 10 +- sync/src/snapshot.rs | 1 + sync/src/tests/snapshot.rs | 1 + util/Cargo.toml | 2 +- util/src/journaldb/overlayrecentdb.rs | 7 +- 21 files changed, 225 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6141f7e31..6b5af327a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "parity" -version = "1.5.11" +version = "1.5.12" dependencies = [ "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -23,7 +23,7 @@ dependencies = [ "ethcore-rpc 1.5.0", "ethcore-signer 1.5.0", "ethcore-stratum 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethsync 1.5.0", "evmbin 1.5.0", "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -303,6 +303,11 @@ name = "dtoa" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "either" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "elastic-array" version = "0.6.0" @@ -373,13 +378,14 @@ dependencies = [ "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", "ethcore-ipc-nano 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethjson 0.1.0", "ethkey 0.2.0", "ethstore 0.1.0", "evmjit 1.5.0", "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)", + "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -421,7 +427,7 @@ dependencies = [ "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.5.0", "ethcore-rpc 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "fetch 0.1.0", "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)", @@ -469,7 +475,7 @@ name = "ethcore-ipc" version = "1.5.0" dependencies = [ "ethcore-devtools 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -516,7 +522,7 @@ dependencies = [ "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", "ethcore-ipc-nano 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -531,7 +537,7 @@ dependencies = [ "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", "ethcore-network 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.1.0", "smallvec 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -543,7 +549,7 @@ name = "ethcore-logger" version = "1.5.0" dependencies = [ "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -559,7 +565,7 @@ dependencies = [ "bytes 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.5.0", "ethcore-io 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethcrypto 0.1.0", "ethkey 0.2.0", "igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -586,7 +592,7 @@ dependencies = [ "ethcore-devtools 1.5.0", "ethcore-io 1.5.0", "ethcore-ipc 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethcrypto 0.1.0", "ethjson 0.1.0", "ethkey 0.2.0", @@ -621,7 +627,7 @@ dependencies = [ "ethcore-devtools 1.5.0", "ethcore-io 1.5.0", "ethcore-rpc 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-dapps-glue 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -640,7 +646,7 @@ dependencies = [ "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", "ethcore-ipc-nano 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)", "jsonrpc-tcp-server 0.1.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -651,7 +657,7 @@ dependencies = [ [[package]] name = "ethcore-util" -version = "1.5.11" +version = "1.5.12" dependencies = [ "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -663,7 +669,7 @@ dependencies = [ "ethcore-bloom-journal 0.1.0", "ethcore-devtools 1.5.0", "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -700,7 +706,7 @@ dependencies = [ name = "ethjson" version = "0.1.0" dependencies = [ - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -727,7 +733,7 @@ dependencies = [ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)", "ethcrypto 0.1.0", "ethkey 0.2.0", - "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -756,7 +762,7 @@ dependencies = [ "ethcore-ipc-nano 1.5.0", "ethcore-light 1.5.0", "ethcore-network 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethkey 0.2.0", "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -773,7 +779,7 @@ version = "1.5.0" dependencies = [ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -975,7 +981,7 @@ version = "1.5.0" dependencies = [ "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -992,9 +998,17 @@ dependencies = [ [[package]] name = "itertools" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "itertools" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itoa" version = "0.1.1" @@ -1500,7 +1514,7 @@ name = "parity-hash-fetch" version = "1.5.0" dependencies = [ "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "fetch 0.1.0", "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1525,7 +1539,7 @@ version = "1.4.0" dependencies = [ "ethcore-rpc 1.5.0", "ethcore-signer 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1571,7 +1585,7 @@ dependencies = [ "ethcore 1.5.0", "ethcore-ipc 1.5.0", "ethcore-ipc-codegen 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "ethsync 1.5.0", "ipc-common-types 1.5.0", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1851,7 +1865,7 @@ version = "1.4.0" dependencies = [ "ethcore-bigint 0.1.2", "ethcore-rpc 1.5.0", - "ethcore-util 1.5.11", + "ethcore-util 1.5.12", "futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-rpc-client 1.4.0", "rpassword 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2469,6 +2483,7 @@ dependencies = [ "checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76" "checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d" "checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f" +"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a" "checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "" "checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5" "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "" @@ -2491,7 +2506,8 @@ dependencies = [ "checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11" "checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484" "checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c" -"checksum itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "086e1fa5fe48840b1cfdef3a20c7e3115599f8d5c4c87ef32a794a7cdd184d76" +"checksum itertools 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)" = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" +"checksum itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d95557e7ba6b71377b0f2c3b3ae96c53f1b75a926a6901a500f557a370af730a" "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1" "checksum itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91fd9dc2c587067de817fec4ad355e3818c3d893a78cab32a0a474c7a15bb8d5" "checksum jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git?branch=mio-old)" = "" diff --git a/Cargo.toml b/Cargo.toml index 0e960f6f2..e503bda14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Parity Ethereum client" name = "parity" -version = "1.5.11" +version = "1.5.12" license = "GPL-3.0" authors = ["Parity Technologies "] build = "build.rs" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index a815bddd6..6388f97d0 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -28,6 +28,7 @@ rand = "0.3" byteorder = "0.5" transient-hashmap = "0.1" linked-hash-map = "0.3.0" +itertools = "0.5" evmjit = { path = "../evmjit", optional = true } clippy = { version = "0.0.103", optional = true} ethash = { path = "../ethash" } diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index df85ca4c7..2279de1ee 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -103,6 +103,7 @@ extern crate ethcore_bloom_journal as bloom_journal; extern crate byteorder; extern crate transient_hashmap; extern crate linked_hash_map; +extern crate itertools; #[macro_use] extern crate log; diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 9d9aa0b9e..a2f789d9d 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -23,6 +23,7 @@ use snapshot::Error; use util::{U256, FixedHash, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP}; use util::trie::{TrieDB, Trie}; use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use itertools::Itertools; use std::collections::HashSet; @@ -60,55 +61,53 @@ impl CodeState { } } -// walk the account's storage trie, returning an RLP item containing the -// account properties and the storage. -pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet) -> Result { +// walk the account's storage trie, returning a vector of RLP items containing the +// account properties and the storage. Each item contains at most `max_storage_items` +// storage records split according to snapshot format definition. +pub fn to_fat_rlps(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet, max_storage_items: usize) -> Result, Error> { if acc == &ACC_EMPTY { - return Ok(::rlp::NULL_RLP.to_vec()); + return Ok(vec![::rlp::NULL_RLP.to_vec()]); } let db = TrieDB::new(acct_db, &acc.storage_root)?; - let mut pairs = Vec::new(); + let chunks = db.iter()?.chunks(max_storage_items); + let pair_chunks = chunks.into_iter().map(|chunk| chunk.collect()); + pair_chunks.pad_using(1, |_| Vec::new(), ).map(|pairs| { + let mut stream = RlpStream::new_list(pairs.len()); - for item in db.iter()? { - let (k, v) = item?; - pairs.push((k, v)); - } + for r in pairs { + let (k, v) = r?; + stream.begin_list(2).append(&k).append(&&*v); + } - let mut stream = RlpStream::new_list(pairs.len()); + let pairs_rlp = stream.out(); - for (k, v) in pairs { - stream.begin_list(2).append(&k).append(&&*v); - } + let mut account_stream = RlpStream::new_list(5); + account_stream.append(&acc.nonce) + .append(&acc.balance); - let pairs_rlp = stream.out(); - - let mut account_stream = RlpStream::new_list(5); - account_stream.append(&acc.nonce) - .append(&acc.balance); - - // [has_code, code_hash]. - if acc.code_hash == SHA3_EMPTY { - account_stream.append(&CodeState::Empty.raw()).append_empty_data(); - } else if used_code.contains(&acc.code_hash) { - account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash); - } else { - match acct_db.get(&acc.code_hash) { - Some(c) => { - used_code.insert(acc.code_hash.clone()); - account_stream.append(&CodeState::Inline.raw()).append(&&*c); - } - None => { - warn!("code lookup failed during snapshot"); - account_stream.append(&false).append_empty_data(); + // [has_code, code_hash]. + if acc.code_hash == SHA3_EMPTY { + account_stream.append(&CodeState::Empty.raw()).append_empty_data(); + } else if used_code.contains(&acc.code_hash) { + account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash); + } else { + match acct_db.get(&acc.code_hash) { + Some(c) => { + used_code.insert(acc.code_hash.clone()); + account_stream.append(&CodeState::Inline.raw()).append(&&*c); + } + None => { + warn!("code lookup failed during snapshot"); + account_stream.append(&false).append_empty_data(); + } } } - } - account_stream.append_raw(&pairs_rlp, 1); - - Ok(account_stream.out()) + account_stream.append_raw(&pairs_rlp, 1); + Ok(account_stream.out()) + }).collect() } // decode a fat rlp, and rebuild the storage trie as we go. @@ -117,6 +116,7 @@ pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashS pub fn from_fat_rlp( acct_db: &mut AccountDBMut, rlp: UntrustedRlp, + mut storage_root: H256, ) -> Result<(BasicAccount, Option), Error> { use util::{TrieDBMut, TrieMut}; @@ -148,10 +148,12 @@ pub fn from_fat_rlp( } }; - let mut storage_root = H256::zero(); - { - let mut storage_trie = TrieDBMut::new(acct_db, &mut storage_root); + let mut storage_trie = if storage_root.is_zero() { + TrieDBMut::new(acct_db, &mut storage_root) + } else { + TrieDBMut::from_existing(acct_db, &mut storage_root)? + }; let pairs = rlp.at(4)?; for pair_rlp in pairs.iter() { let k: Bytes = pair_rlp.val_at(0)?; @@ -184,7 +186,7 @@ mod tests { use std::collections::HashSet; - use super::{ACC_EMPTY, to_fat_rlp, from_fat_rlp}; + use super::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; #[test] fn encoding_basic() { @@ -201,9 +203,9 @@ mod tests { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp), account); - let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap(); - let fat_rlp = UntrustedRlp::new(&fat_rlp); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account); + let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap(); + let fat_rlp = UntrustedRlp::new(&fat_rlps[0]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account); } #[test] @@ -226,9 +228,40 @@ mod tests { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp), account); - let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap(); - let fat_rlp = UntrustedRlp::new(&fat_rlp); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account); + let fat_rlp = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap(); + let fat_rlp = UntrustedRlp::new(&fat_rlp[0]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account); + } + + #[test] + fn encoding_storage_split() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = { + let acct_db = AccountDBMut::new(db.as_hashdb_mut(), &addr); + let mut root = SHA3_NULL_RLP; + fill_storage(acct_db, &mut root, &mut H256::zero()); + BasicAccount { + nonce: 25.into(), + balance: 987654321.into(), + storage_root: root, + code_hash: SHA3_EMPTY, + } + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp), account); + + let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), 100).unwrap(); + let mut root = SHA3_NULL_RLP; + let mut restored_account = None; + for rlp in fat_rlps { + let fat_rlp = UntrustedRlp::new(&rlp); + restored_account = Some(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, root).unwrap().0); + root = restored_account.as_ref().unwrap().storage_root.clone(); + } + assert_eq!(restored_account, Some(account)); } #[test] @@ -264,18 +297,18 @@ mod tests { let mut used_code = HashSet::new(); - let fat_rlp1 = to_fat_rlp(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code).unwrap(); - let fat_rlp2 = to_fat_rlp(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code).unwrap(); + let fat_rlp1 = to_fat_rlps(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code, usize::max_value()).unwrap(); + let fat_rlp2 = to_fat_rlps(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code, usize::max_value()).unwrap(); assert_eq!(used_code.len(), 1); - let fat_rlp1 = UntrustedRlp::new(&fat_rlp1); - let fat_rlp2 = UntrustedRlp::new(&fat_rlp2); + let fat_rlp1 = UntrustedRlp::new(&fat_rlp1[0]); + let fat_rlp2 = UntrustedRlp::new(&fat_rlp2[0]); - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2).unwrap(); + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2, H256::zero()).unwrap(); assert!(maybe_code.is_none()); assert_eq!(acc, account2); - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1).unwrap(); + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1, H256::zero()).unwrap(); assert_eq!(maybe_code, Some(b"this is definitely code".to_vec())); assert_eq!(acc, account1); } @@ -285,7 +318,7 @@ mod tests { let mut db = get_temp_state_db(); let mut used_code = HashSet::new(); - assert_eq!(to_fat_rlp(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code).unwrap(), ::rlp::NULL_RLP.to_vec()); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP)).unwrap(), (ACC_EMPTY, None)); + assert_eq!(to_fat_rlps(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code, usize::max_value()).unwrap(), vec![::rlp::NULL_RLP.to_vec()]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP), H256::zero()).unwrap(), (ACC_EMPTY, None)); } } diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 46ef6f41b..0e71217c2 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -53,6 +53,8 @@ pub enum Error { Decoder(DecoderError), /// Io error. Io(::std::io::Error), + /// Snapshot version is not supported. + VersionNotSupported(u64), } impl fmt::Display for Error { @@ -73,6 +75,7 @@ impl fmt::Display for Error { Error::Io(ref err) => err.fmt(f), Error::Decoder(ref err) => err.fmt(f), Error::Trie(ref err) => err.fmt(f), + Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver), } } } diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 9ef2ad5cb..309d5e4d4 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -31,6 +31,8 @@ use rlp::{self, Encodable, RlpStream, UntrustedRlp, Stream, View}; use super::ManifestData; +const SNAPSHOT_VERSION: u64 = 2; + /// Something which can write snapshots. /// Writing the same chunk multiple times will lead to implementation-defined /// behavior, and is not advised. @@ -120,8 +122,9 @@ impl SnapshotWriter for PackedWriter { fn finish(mut self, manifest: ManifestData) -> io::Result<()> { // we ignore the hashes fields of the manifest under the assumption that // they are consistent with ours. - let mut stream = RlpStream::new_list(5); + let mut stream = RlpStream::new_list(6); stream + .append(&SNAPSHOT_VERSION) .append(&self.state_hashes) .append(&self.block_hashes) .append(&manifest.state_root) @@ -223,7 +226,7 @@ impl PackedReader { /// Create a new `PackedReader` for the file at the given path. /// This will fail if any io errors are encountered or the file /// is not a valid packed snapshot. - pub fn new(path: &Path) -> Result, ::error::Error> { + pub fn new(path: &Path) -> Result, ::snapshot::error::Error> { let mut file = File::open(path)?; let file_len = file.metadata()?.len(); if file_len < 8 { @@ -257,15 +260,26 @@ impl PackedReader { let rlp = UntrustedRlp::new(&manifest_buf); - let state: Vec = rlp.val_at(0)?; - let blocks: Vec = rlp.val_at(1)?; + let (start, version) = if rlp.item_count() == 5 { + (0, 1) + } else { + (1, rlp.val_at(0)?) + }; + + if version > SNAPSHOT_VERSION { + return Err(::snapshot::error::Error::VersionNotSupported(version)); + } + + let state: Vec = rlp.val_at(0 + start)?; + let blocks: Vec = rlp.val_at(1 + start)?; let manifest = ManifestData { + version: version, state_hashes: state.iter().map(|c| c.0).collect(), block_hashes: blocks.iter().map(|c| c.0).collect(), - state_root: rlp.val_at(2)?, - block_number: rlp.val_at(3)?, - block_hash: rlp.val_at(4)?, + state_root: rlp.val_at(2 + start)?, + block_number: rlp.val_at(3 + start)?, + block_hash: rlp.val_at(4 + start)?, }; Ok(Some(PackedReader { @@ -348,7 +362,7 @@ mod tests { use util::sha3::Hashable; use snapshot::ManifestData; - use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader}; + use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; const BLOCK_CHUNKS: &'static [&'static [u8]] = &[b"hello!", b"goodbye!", b"abcdefg", b"hijklmnop", b"qrstuvwxy", b"and", b"z"]; @@ -374,6 +388,7 @@ mod tests { } let manifest = ManifestData { + version: SNAPSHOT_VERSION, state_hashes: state_hashes, block_hashes: block_hashes, state_root: b"notarealroot".sha3(), @@ -412,6 +427,7 @@ mod tests { } let manifest = ManifestData { + version: SNAPSHOT_VERSION, state_hashes: state_hashes, block_hashes: block_hashes, state_root: b"notarealroot".sha3(), @@ -428,4 +444,4 @@ mod tests { reader.chunk(hash.clone()).unwrap(); } } -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index add5029b2..8b8f2e6d2 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -56,6 +56,7 @@ pub use self::traits::SnapshotService; pub use self::watcher::Watcher; pub use types::snapshot_manifest::ManifestData; pub use types::restoration_status::RestorationStatus; +pub use types::basic_account::BasicAccount; pub mod io; pub mod service; @@ -82,6 +83,9 @@ mod traits { // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; +// Try to have chunks be around 4MB (before compression) +const MAX_STORAGE_ENTRIES_PER_ACCOUNT_RECORD: usize = 80_000; + // How many blocks to include in a snapshot, starting from the head of the chain. const SNAPSHOT_BLOCKS: u64 = 30000; @@ -147,6 +151,7 @@ pub fn take_snapshot( info!("produced {} state chunks and {} block chunks.", state_hashes.len(), block_hashes.len()); let manifest_data = ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: block_hashes, state_root: *state_root, @@ -300,14 +305,14 @@ impl<'a> StateChunker<'a> { // // If the buffer is greater than the desired chunk size, // this will write out the data to disk. - fn push(&mut self, account_hash: Bytes, data: Bytes) -> Result<(), Error> { + fn push(&mut self, account_hash: Bytes, data: Bytes, force_chunk: bool) -> Result<(), Error> { let pair = { let mut stream = RlpStream::new_list(2); stream.append(&account_hash).append_raw(&data, 1); stream.out() }; - if self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE { + if force_chunk || self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE { self.write_chunk()?; } @@ -372,8 +377,10 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex 0)?; + } } if chunker.cur_size != 0 { @@ -390,6 +397,7 @@ pub struct StateRebuilder { known_code: HashMap, // code hashes mapped to first account with this code. missing_code: HashMap>, // maps code hashes to lists of accounts missing that code. bloom: Bloom, + known_storage_roots: HashMap, // maps account hashes to last known storage root. Only filled for last account per chunk. } impl StateRebuilder { @@ -401,6 +409,7 @@ impl StateRebuilder { known_code: HashMap::new(), missing_code: HashMap::new(), bloom: StateDB::load_bloom(&*db), + known_storage_roots: HashMap::new(), } } @@ -418,6 +427,7 @@ impl StateRebuilder { rlp, &mut pairs, &self.known_code, + &mut self.known_storage_roots, flag )?; @@ -492,10 +502,11 @@ fn rebuild_accounts( account_fat_rlps: UntrustedRlp, out_chunk: &mut [(H256, Bytes)], known_code: &HashMap, + known_storage_roots: &mut HashMap, abort_flag: &AtomicBool, ) -> Result { let mut status = RebuiltStatus::default(); - for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk) { + for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk.iter_mut()) { if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) } let hash: H256 = account_rlp.val_at(0)?; @@ -506,7 +517,8 @@ fn rebuild_accounts( // fill out the storage trie and code while decoding. let (acc, maybe_code) = { let mut acct_db = AccountDBMut::from_hash(db, hash); - account::from_fat_rlp(&mut acct_db, fat_rlp)? + let storage_root = known_storage_roots.get(&hash).cloned().unwrap_or(H256::zero()); + account::from_fat_rlp(&mut acct_db, fat_rlp, storage_root)? }; let code_hash = acc.code_hash.clone(); @@ -538,6 +550,12 @@ fn rebuild_accounts( *out = (hash, thin_rlp); } + if let Some(&(ref hash, ref rlp)) = out_chunk.iter().last() { + known_storage_roots.insert(*hash, ::rlp::decode::(rlp).storage_root); + } + if let Some(&(ref hash, ref rlp)) = out_chunk.iter().next() { + known_storage_roots.insert(*hash, ::rlp::decode::(rlp).storage_root); + } Ok(status) } diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index 8d0d14ac7..e4f739260 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -658,6 +658,7 @@ mod tests { assert_eq!(service.status(), RestorationStatus::Inactive); let manifest = ManifestData { + version: 2, state_hashes: vec![], block_hashes: vec![], state_root: Default::default(), diff --git a/ethcore/src/snapshot/tests/blocks.rs b/ethcore/src/snapshot/tests/blocks.rs index 058d311e2..64b7661e9 100644 --- a/ethcore/src/snapshot/tests/blocks.rs +++ b/ethcore/src/snapshot/tests/blocks.rs @@ -63,6 +63,7 @@ fn chunk_and_restore(amount: u64) { let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); let block_hashes = chunk_blocks(&bc, best_hash, &writer, &Progress::default()).unwrap(); let manifest = ::snapshot::ManifestData { + version: 2, state_hashes: Vec::new(), block_hashes: block_hashes, state_root: ::util::sha3::SHA3_NULL_RLP, @@ -125,6 +126,7 @@ fn checks_flag() { let chain = BlockChain::new(Default::default(), &genesis, db.clone(), engine.clone()); let manifest = ::snapshot::ManifestData { + version: 2, state_hashes: Vec::new(), block_hashes: Vec::new(), state_root: ::util::sha3::SHA3_NULL_RLP, diff --git a/ethcore/src/snapshot/tests/mod.rs b/ethcore/src/snapshot/tests/mod.rs index d63abec53..52fc62ba5 100644 --- a/ethcore/src/snapshot/tests/mod.rs +++ b/ethcore/src/snapshot/tests/mod.rs @@ -27,6 +27,7 @@ use super::ManifestData; #[test] fn manifest_rlp() { let manifest = ManifestData { + version: 2, block_hashes: Vec::new(), state_hashes: Vec::new(), block_number: 1234567, @@ -35,4 +36,4 @@ fn manifest_rlp() { }; let raw = manifest.clone().into_rlp(); assert_eq!(ManifestData::from_rlp(&raw).unwrap(), manifest); -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/src/snapshot/tests/service.rs index b460d5806..97ad051d0 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/src/snapshot/tests/service.rs @@ -122,6 +122,7 @@ fn guards_delete_folders() { path.push("restoration"); let manifest = ManifestData { + version: 2, state_hashes: vec![], block_hashes: vec![], block_number: 0, diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index 5fcc5a52c..f3815ab75 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -58,6 +58,7 @@ fn snap_and_restore() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap(); writer.into_inner().finish(::snapshot::ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: Vec::new(), state_root: state_root, @@ -120,10 +121,10 @@ fn get_code_from_prev_chunk() { let mut db = MemoryDB::new(); AccountDBMut::from_hash(&mut db, hash).insert(&code[..]); - let fat_rlp = account::to_fat_rlp(&acc, &AccountDB::from_hash(&db, hash), &mut used_code).unwrap(); + let fat_rlp = account::to_fat_rlps(&acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value()).unwrap(); let mut stream = RlpStream::new_list(1); - stream.begin_list(2).append(&hash).append_raw(&fat_rlp, 1); + stream.begin_list(2).append(&hash).append_raw(&fat_rlp[0], 1); stream.out() }; @@ -164,6 +165,7 @@ fn checks_flag() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap(); writer.into_inner().finish(::snapshot::ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: Vec::new(), state_root: state_root, diff --git a/ethcore/src/types/snapshot_manifest.rs b/ethcore/src/types/snapshot_manifest.rs index f3f189aed..a5bc4bae3 100644 --- a/ethcore/src/types/snapshot_manifest.rs +++ b/ethcore/src/types/snapshot_manifest.rs @@ -24,6 +24,8 @@ use util::Bytes; #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "ipc", binary)] pub struct ManifestData { + /// Snapshot format version. + pub version: u64, /// List of state chunk hashes. pub state_hashes: Vec, /// List of block chunk hashes. @@ -39,7 +41,8 @@ pub struct ManifestData { impl ManifestData { /// Encode the manifest data to rlp. pub fn into_rlp(self) -> Bytes { - let mut stream = RlpStream::new_list(5); + let mut stream = RlpStream::new_list(6); + stream.append(&self.version); stream.append(&self.state_hashes); stream.append(&self.block_hashes); stream.append(&self.state_root); @@ -52,14 +55,20 @@ impl ManifestData { /// Try to restore manifest data from raw bytes, interpreted as RLP. pub fn from_rlp(raw: &[u8]) -> Result { let decoder = UntrustedRlp::new(raw); + let (start, version) = if decoder.item_count() == 5 { + (0, 1) + } else { + (1, decoder.val_at(0)?) + }; - let state_hashes: Vec = decoder.val_at(0)?; - let block_hashes: Vec = decoder.val_at(1)?; - let state_root: H256 = decoder.val_at(2)?; - let block_number: u64 = decoder.val_at(3)?; - let block_hash: H256 = decoder.val_at(4)?; + let state_hashes: Vec = decoder.val_at(start + 0)?; + let block_hashes: Vec = decoder.val_at(start + 1)?; + let state_root: H256 = decoder.val_at(start + 2)?; + let block_number: u64 = decoder.val_at(start + 3)?; + let block_hash: H256 = decoder.val_at(start + 4)?; Ok(ManifestData { + version: version, state_hashes: state_hashes, block_hashes: block_hashes, state_root: state_root, diff --git a/mac/Parity.pkgproj b/mac/Parity.pkgproj index 6da6a0aef..177f50dea 100755 --- a/mac/Parity.pkgproj +++ b/mac/Parity.pkgproj @@ -578,7 +578,7 @@ OVERWRITE_PERMISSIONS VERSION - 1.5.11 + 1.5.12 UUID 2DCD5B81-7BAF-4DA1-9251-6274B089FD36 diff --git a/nsis/installer.nsi b/nsis/installer.nsi index e2c1b2db9..43d8d9a6d 100644 --- a/nsis/installer.nsi +++ b/nsis/installer.nsi @@ -10,7 +10,7 @@ !define DESCRIPTION "Fast, light, robust Ethereum implementation" !define VERSIONMAJOR 1 !define VERSIONMINOR 5 -!define VERSIONBUILD 11 +!define VERSIONBUILD 12 !define ARGS "--warp" !define FIRST_START_ARGS "ui --warp --mode=passive" diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 2db0b2ae6..35532042f 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -158,6 +158,8 @@ pub const SNAPSHOT_SYNC_PACKET_COUNT: u8 = 0x16; const MAX_SNAPSHOT_CHUNKS_DOWNLOAD_AHEAD: usize = 3; +const MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION: u64 = 1; + const WAIT_PEERS_TIMEOUT_SEC: u64 = 5; const STATUS_TIMEOUT_SEC: u64 = 5; const HEADERS_TIMEOUT_SEC: u64 = 15; @@ -1033,12 +1035,18 @@ impl ChainSync { let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) { Err(e) => { trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e); - io.disconnect_peer(peer_id); + io.disable_peer(peer_id); self.continue_sync(io); return Ok(()); } Ok(manifest) => manifest, }; + if manifest.version < MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION { + trace!(target: "sync", "{}: Snapshot manifest version too low: {}", peer_id, manifest.version); + io.disable_peer(peer_id); + self.continue_sync(io); + return Ok(()); + } self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3()); io.snapshot_service().begin_restore(manifest); self.state = SyncState::SnapshotData; diff --git a/sync/src/snapshot.rs b/sync/src/snapshot.rs index 7ec87bf01..cb2dc062e 100644 --- a/sync/src/snapshot.rs +++ b/sync/src/snapshot.rs @@ -139,6 +139,7 @@ mod test { let state_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); let block_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); let manifest = ManifestData { + version: 2, state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), state_root: H256::new(), diff --git a/sync/src/tests/snapshot.rs b/sync/src/tests/snapshot.rs index 96a801669..37ddab260 100644 --- a/sync/src/tests/snapshot.rs +++ b/sync/src/tests/snapshot.rs @@ -49,6 +49,7 @@ impl TestSnapshotService { let state_chunks: Vec = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect(); let block_chunks: Vec = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect(); let manifest = ManifestData { + version: 2, state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), state_root: H256::new(), diff --git a/util/Cargo.toml b/util/Cargo.toml index 262c23a4d..4ab0e536b 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -3,7 +3,7 @@ description = "Ethcore utility library" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore-util" -version = "1.5.11" +version = "1.5.12" authors = ["Parity Technologies "] build = "build.rs" diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index db320f07d..15eb2843d 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -347,10 +347,7 @@ impl JournalDB for OverlayRecentDB { match rc { 0 => {} - 1 => { - if cfg!(debug_assertions) && self.backing.get(self.column, &key)?.is_some() { - return Err(BaseDataError::AlreadyExists(key).into()); - } + _ if rc > 0 => { batch.put(self.column, &key, &value) } -1 => { @@ -359,7 +356,7 @@ impl JournalDB for OverlayRecentDB { } batch.delete(self.column, &key) } - _ => panic!("Attempted to inject invalid state."), + _ => panic!("Attempted to inject invalid state ({})", rc), } }