From 797f23f30b1657eb4c6dece36cad75663de47d24 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Tue, 28 Mar 2017 01:45:56 +0200 Subject: [PATCH] [beta] Backports (#5299) * Fix FireFox overflows (#5000) * Max width for container * Set min-width * Switching ValidatorSet (#4961) * add multi validator set * nicer comment * validate in constructor * reporting * Avoid clogging up tmp when updater dir has bad permissions. (#5024) * force earliest era set in snapshot restore (#5021) * v1.6.5 * Fine grained snapshot chunking * Ropsten revival --- Cargo.lock | 51 +++---- Cargo.toml | 2 +- ethcore/Cargo.toml | 1 + ethcore/res/ethereum/ropsten.json | 11 +- ethcore/res/validator_multi.json | 42 ++++++ ethcore/src/engines/authority_round.rs | 2 +- ethcore/src/engines/basic_authority.rs | 2 +- ethcore/src/engines/tendermint/mod.rs | 2 +- ethcore/src/engines/validator_set/mod.rs | 15 +- ethcore/src/engines/validator_set/multi.rs | 158 +++++++++++++++++++++ ethcore/src/lib.rs | 1 + ethcore/src/snapshot/account.rs | 143 ++++++++++++------- ethcore/src/snapshot/error.rs | 3 + ethcore/src/snapshot/io.rs | 34 +++-- ethcore/src/snapshot/mod.rs | 48 +++++-- ethcore/src/snapshot/service.rs | 3 +- ethcore/src/snapshot/tests/blocks.rs | 2 + ethcore/src/snapshot/tests/mod.rs | 3 +- ethcore/src/snapshot/tests/service.rs | 1 + ethcore/src/snapshot/tests/state.rs | 32 +++-- ethcore/src/spec/spec.rs | 4 + ethcore/src/types/snapshot_manifest.rs | 21 ++- js/src/ui/Container/container.css | 1 + js/src/ui/SectionList/sectionList.css | 2 + json/src/spec/validator_set.rs | 17 ++- mac/Parity.pkgproj | 2 +- mac/Parity/Info.plist | 2 +- nsis/installer.nsi | 2 +- sync/src/chain.rs | 10 +- sync/src/snapshot.rs | 1 + sync/src/tests/snapshot.rs | 1 + updater/src/updater.rs | 2 +- util/Cargo.toml | 2 +- util/src/journaldb/overlayrecentdb.rs | 7 +- 34 files changed, 478 insertions(+), 152 deletions(-) create mode 100644 ethcore/res/validator_multi.json create mode 100644 ethcore/src/engines/validator_set/multi.rs diff --git a/Cargo.lock b/Cargo.lock index 71c889331..93d03a2e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "parity" -version = "1.6.4" +version = "1.6.5" dependencies = [ "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -23,7 +23,7 @@ dependencies = [ "ethcore-secretstore 1.0.0", "ethcore-signer 1.6.0", "ethcore-stratum 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethsync 1.6.0", "evmbin 0.1.0", "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -394,13 +394,14 @@ dependencies = [ "ethcore-ipc-codegen 1.6.0", "ethcore-ipc-nano 1.6.0", "ethcore-stratum 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethjson 0.1.0", "ethkey 0.2.0", "ethstore 0.1.0", "evmjit 1.6.0", "hardware-wallet 1.6.0", "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)", + "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -443,7 +444,7 @@ dependencies = [ "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.6.0", "ethcore-rpc 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "fetch 0.1.0", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)", @@ -491,7 +492,7 @@ name = "ethcore-ipc" version = "1.6.0" dependencies = [ "ethcore-devtools 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git?branch=parity-1.7)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -538,7 +539,7 @@ dependencies = [ "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", "ethcore-ipc-nano 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git?branch=parity-1.7)", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -553,7 +554,7 @@ dependencies = [ "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", "ethcore-network 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -569,7 +570,7 @@ name = "ethcore-logger" version = "1.6.0" dependencies = [ "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -585,7 +586,7 @@ dependencies = [ "bytes 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.6.0", "ethcore-io 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethcrypto 0.1.0", "ethkey 0.2.0", "igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -613,7 +614,7 @@ dependencies = [ "ethcore-io 1.6.0", "ethcore-ipc 1.6.0", "ethcore-light 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethcrypto 0.1.0", "ethjson 0.1.0", "ethkey 0.2.0", @@ -648,7 +649,7 @@ dependencies = [ "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", "ethcore-ipc-nano 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethcrypto 0.1.0", "ethkey 0.2.0", "hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -666,7 +667,7 @@ dependencies = [ "ethcore-devtools 1.6.0", "ethcore-io 1.6.0", "ethcore-rpc 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-dapps-glue 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -685,7 +686,7 @@ dependencies = [ "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", "ethcore-ipc-nano 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)", "jsonrpc-macros 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)", @@ -699,7 +700,7 @@ dependencies = [ [[package]] name = "ethcore-util" -version = "1.6.4" +version = "1.6.5" dependencies = [ "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -748,7 +749,7 @@ dependencies = [ name = "ethjson" version = "0.1.0" dependencies = [ - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -776,7 +777,7 @@ version = "0.1.0" dependencies = [ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethcrypto 0.1.0", "ethkey 0.2.0", "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -809,7 +810,7 @@ dependencies = [ "ethcore-ipc-nano 1.6.0", "ethcore-light 1.6.0", "ethcore-network 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethkey 0.2.0", "heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -826,7 +827,7 @@ version = "0.1.0" dependencies = [ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1054,7 +1055,7 @@ version = "1.6.0" dependencies = [ "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1621,7 +1622,7 @@ name = "parity-hash-fetch" version = "1.6.0" dependencies = [ "ethabi 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "fetch 0.1.0", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1638,7 +1639,7 @@ version = "1.6.0" dependencies = [ "cid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)", "jsonrpc-http-server 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)", "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1652,7 +1653,7 @@ version = "0.1.0" dependencies = [ "ethcore 1.6.0", "ethcore-io 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethkey 0.2.0", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.1.0", @@ -1675,7 +1676,7 @@ version = "1.4.0" dependencies = [ "ethcore-rpc 1.6.0", "ethcore-signer 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 6.0.0 (git+https://github.com/ethcore/jsonrpc.git)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1719,7 +1720,7 @@ dependencies = [ "ethcore 1.6.0", "ethcore-ipc 1.6.0", "ethcore-ipc-codegen 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "ethsync 1.6.0", "ipc-common-types 1.6.0", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1994,7 +1995,7 @@ version = "1.4.0" dependencies = [ "ethcore-bigint 0.1.2", "ethcore-rpc 1.6.0", - "ethcore-util 1.6.4", + "ethcore-util 1.6.5", "futures 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "parity-rpc-client 1.4.0", "rpassword 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 7b93b46d0..1dbcba4d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Parity Ethereum client" name = "parity" -version = "1.6.4" +version = "1.6.5" license = "GPL-3.0" authors = ["Parity Technologies "] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index c8a1c7fb5..9a86663de 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -27,6 +27,7 @@ byteorder = "1.0" transient-hashmap = "0.1" linked-hash-map = "0.3.0" lru-cache = "0.1.0" +itertools = "0.5" ethabi = "1.0.0" evmjit = { path = "../evmjit", optional = true } clippy = { version = "0.0.103", optional = true} diff --git a/ethcore/res/ethereum/ropsten.json b/ethcore/res/ethereum/ropsten.json index 231073e4d..539168279 100644 --- a/ethcore/res/ethereum/ropsten.json +++ b/ethcore/res/ethereum/ropsten.json @@ -25,8 +25,8 @@ "maximumExtraDataSize": "0x20", "minGasLimit": "0x1388", "networkID" : "0x3", - "forkBlock": 333922, - "forkCanonHash": "0x8737eb141d4f05db57af63fc8d3b4d4d8f9cddb0c4e1ab855de8c288fdc1924f", + "forkBlock": 641350, + "forkCanonHash": "0x8033403e9fe5811a7b6d6b469905915de1c59207ce2172cbcf5d6ff14fa6a2eb", "eip98Transition": "0x7fffffffffffff" }, "genesis": { @@ -44,11 +44,8 @@ "gasLimit": "0x1000000" }, "nodes": [ - "enode://a22f0977ce02653bf95e38730106356342df48b5222e2c2a1a6f9ef34769bf593bae9ca0a888cf60839edd52efc1b6e393c63a57d76f4c4fe14e641f1f9e637e@128.199.55.137:30303", - "enode://012239fccf3ff1d92b036983a430cb6705c6528c96c0354413f8854802138e5135c084ab36e7c54efb621c46728df8c3a6f4c1db9bb48a1330efe3f82f2dd7a6@52.169.94.142:30303", - "enode://1462682e4b7ba2258346d55e25e5b9d264b0db40cee12bdfba4e72b1d7050350ea954c006e9106dd96a128e6e0bd6dffb17eed51f9f99bf7f9cdadfeaf8da4ff@51.15.61.253:30303", - "enode://98fbb020c799ae39a828bd75dc2bd5d4721539faf317076b275f91182a5c8900b592e8abfdddceae674a7c3bb40ea00a6ca9ccb7805ab58c4b7b29c61c8f7239@51.15.62.44:30303", - "enode://d801dd4e3d15a8bf785931add164bd9c313e3f6b5749d9302b311f2b48064cba5c86c32b1302c27cd983fc89ae07d4d306dd1197610835b8782e95dfb1b3f9ea@51.15.43.255:30303" + "enode://20c9ad97c081d63397d7b685a412227a40e23c8bdc6688c6f37e97cfbc22d2b4d1db1510d8f61e6a8866ad7f0e17c02b14182d37ea7c3c8b9c2683aeb6b733a1@52.169.14.227:30303", + "enode://6ce05930c72abc632c58e2e4324f7c7ea478cec0ed4fa2528982cf34483094e9cbc9216e7aa349691242576d552a2a56aaeae426c5303ded677ce455ba1acd9d@13.84.180.240:30303" ], "accounts": { "0000000000000000000000000000000000000001": { "balance": "1", "nonce": "0", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, diff --git a/ethcore/res/validator_multi.json b/ethcore/res/validator_multi.json new file mode 100644 index 000000000..5d51b73da --- /dev/null +++ b/ethcore/res/validator_multi.json @@ -0,0 +1,42 @@ +{ + "name": "TestMutiValidator", + "engine": { + "basicAuthority": { + "params": { + "gasLimitBoundDivisor": "0x0400", + "durationLimit": "0x0d", + "validators": { + "multi": { + "0": { "list": ["0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1"] }, + "2": { "list": ["0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e"] } + } + } + } + } + }, + "params": { + "accountStartNonce": "0x0", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID" : "0x69" + }, + "genesis": { + "seal": { + "generic": "0xc180" + }, + "difficulty": "0x20000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x", + "gasLimit": "0x2fefd8" + }, + "accounts": { + "0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1": { "balance": "99999999999999999999999" }, + "0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e": { "balance": "99999999999999999999999" } + } +} diff --git a/ethcore/src/engines/authority_round.rs b/ethcore/src/engines/authority_round.rs index ef1bced4d..59a566572 100644 --- a/ethcore/src/engines/authority_round.rs +++ b/ethcore/src/engines/authority_round.rs @@ -82,7 +82,7 @@ pub struct AuthorityRound { proposed: AtomicBool, client: RwLock>>, signer: EngineSigner, - validators: Box, + validators: Box, /// Is this Engine just for testing (prevents step calibration). calibrate_step: bool, } diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index 34b89b2d6..ff08e3850 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -58,7 +58,7 @@ pub struct BasicAuthority { gas_limit_bound_divisor: U256, builtins: BTreeMap, signer: EngineSigner, - validators: Box, + validators: Box, } impl BasicAuthority { diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index aac101447..56f11d9fe 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -98,7 +98,7 @@ pub struct Tendermint { /// Hash of the proposal parent block. proposal_parent: RwLock, /// Set used to determine the current validators. - validators: Box, + validators: Box, } impl Tendermint { diff --git a/ethcore/src/engines/validator_set/mod.rs b/ethcore/src/engines/validator_set/mod.rs index 3e86c357f..cbbedfb33 100644 --- a/ethcore/src/engines/validator_set/mod.rs +++ b/ethcore/src/engines/validator_set/mod.rs @@ -19,6 +19,7 @@ mod simple_list; mod safe_contract; mod contract; +mod multi; use std::sync::Weak; use util::{Address, H256}; @@ -27,23 +28,27 @@ use client::Client; use self::simple_list::SimpleList; use self::contract::ValidatorContract; use self::safe_contract::ValidatorSafeContract; +use self::multi::Multi; /// Creates a validator set from spec. -pub fn new_validator_set(spec: ValidatorSpec) -> Box { +pub fn new_validator_set(spec: ValidatorSpec) -> Box { match spec { ValidatorSpec::List(list) => Box::new(SimpleList::new(list.into_iter().map(Into::into).collect())), ValidatorSpec::SafeContract(address) => Box::new(ValidatorSafeContract::new(address.into())), ValidatorSpec::Contract(address) => Box::new(ValidatorContract::new(address.into())), + ValidatorSpec::Multi(sequence) => Box::new( + Multi::new(sequence.into_iter().map(|(block, set)| (block.into(), new_validator_set(set))).collect()) + ), } } -pub trait ValidatorSet { +pub trait ValidatorSet: Send + Sync { /// Checks if a given address is a validator. - fn contains(&self, bh: &H256, address: &Address) -> bool; + fn contains(&self, parent_block_hash: &H256, address: &Address) -> bool; /// Draws an validator nonce modulo number of validators. - fn get(&self, bh: &H256, nonce: usize) -> Address; + fn get(&self, parent_block_hash: &H256, nonce: usize) -> Address; /// Returns the current number of validators. - fn count(&self, bh: &H256) -> usize; + fn count(&self, parent_block_hash: &H256) -> usize; /// Notifies about malicious behaviour. fn report_malicious(&self, _validator: &Address) {} /// Notifies about benign misbehaviour. diff --git a/ethcore/src/engines/validator_set/multi.rs b/ethcore/src/engines/validator_set/multi.rs new file mode 100644 index 000000000..5027f23cd --- /dev/null +++ b/ethcore/src/engines/validator_set/multi.rs @@ -0,0 +1,158 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +/// Validator set changing at fork blocks. + +use std::collections::BTreeMap; +use std::sync::Weak; +use util::{H256, Address, RwLock}; +use ids::BlockId; +use header::BlockNumber; +use client::{Client, BlockChainClient}; +use super::ValidatorSet; + +type BlockNumberLookup = Box Result + Send + Sync + 'static>; + +pub struct Multi { + sets: BTreeMap>, + block_number: RwLock, +} + +impl Multi { + pub fn new(set_map: BTreeMap>) -> Self { + assert!(set_map.get(&0u64).is_some(), "ValidatorSet has to be specified from block 0."); + Multi { + sets: set_map, + block_number: RwLock::new(Box::new(move |_| Err("No client!".into()))), + } + } + + fn correct_set(&self, bh: &H256) -> Option<&Box> { + match self + .block_number + .read()(bh) + .map(|parent_block| self + .sets + .iter() + .rev() + .find(|&(block, _)| *block <= parent_block + 1) + .expect("constructor validation ensures that there is at least one validator set for block 0; + block 0 is less than any uint; + qed") + ) { + Ok((block, set)) => { + trace!(target: "engine", "Multi ValidatorSet retrieved for block {}.", block); + Some(set) + }, + Err(e) => { + debug!(target: "engine", "ValidatorSet could not be recovered: {}", e); + None + }, + } + } +} + +impl ValidatorSet for Multi { + fn contains(&self, bh: &H256, address: &Address) -> bool { + self.correct_set(bh).map_or(false, |set| set.contains(bh, address)) + } + + fn get(&self, bh: &H256, nonce: usize) -> Address { + self.correct_set(bh).map_or_else(Default::default, |set| set.get(bh, nonce)) + } + + fn count(&self, bh: &H256) -> usize { + self.correct_set(bh).map_or_else(usize::max_value, |set| set.count(bh)) + } + + fn report_malicious(&self, validator: &Address) { + for set in self.sets.values() { + set.report_malicious(validator); + } + } + + fn report_benign(&self, validator: &Address) { + for set in self.sets.values() { + set.report_benign(validator); + } + } + + fn register_contract(&self, client: Weak) { + for set in self.sets.values() { + set.register_contract(client.clone()); + } + *self.block_number.write() = Box::new(move |hash| client + .upgrade() + .ok_or("No client!".into()) + .and_then(|c| c.block_number(BlockId::Hash(*hash)).ok_or("Unknown block".into()))); + } +} + +#[cfg(test)] +mod tests { + use util::*; + use types::ids::BlockId; + use spec::Spec; + use account_provider::AccountProvider; + use client::{BlockChainClient, EngineClient}; + use ethkey::Secret; + use miner::MinerService; + use tests::helpers::{generate_dummy_client_with_spec_and_accounts, generate_dummy_client_with_spec_and_data}; + + #[test] + fn uses_current_set() { + ::env_logger::init().unwrap(); + let tap = Arc::new(AccountProvider::transient_provider()); + let s0 = Secret::from_slice(&"0".sha3()).unwrap(); + let v0 = tap.insert_account(s0.clone(), "").unwrap(); + let v1 = tap.insert_account(Secret::from_slice(&"1".sha3()).unwrap(), "").unwrap(); + let client = generate_dummy_client_with_spec_and_accounts(Spec::new_validator_multi, Some(tap)); + client.engine().register_client(Arc::downgrade(&client)); + + // Make sure txs go through. + client.miner().set_gas_floor_target(1_000_000.into()); + + // Wrong signer for the first block. + client.miner().set_engine_signer(v1, "".into()).unwrap(); + client.transact_contract(Default::default(), Default::default()).unwrap(); + client.update_sealing(); + assert_eq!(client.chain_info().best_block_number, 0); + // Right signer for the first block. + client.miner().set_engine_signer(v0, "".into()).unwrap(); + client.update_sealing(); + assert_eq!(client.chain_info().best_block_number, 1); + // This time v0 is wrong. + client.transact_contract(Default::default(), Default::default()).unwrap(); + client.update_sealing(); + assert_eq!(client.chain_info().best_block_number, 1); + client.miner().set_engine_signer(v1, "".into()).unwrap(); + client.update_sealing(); + assert_eq!(client.chain_info().best_block_number, 2); + // v1 is still good. + client.transact_contract(Default::default(), Default::default()).unwrap(); + client.update_sealing(); + assert_eq!(client.chain_info().best_block_number, 3); + + // Check syncing. + let sync_client = generate_dummy_client_with_spec_and_data(Spec::new_validator_multi, 0, 0, &[]); + sync_client.engine().register_client(Arc::downgrade(&sync_client)); + for i in 1..4 { + sync_client.import_block(client.block(BlockId::Number(i)).unwrap().into_inner()).unwrap(); + } + sync_client.flush_queue(); + assert_eq!(sync_client.chain_info().best_block_number, 3); + } +} diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 2d3307b66..10436bf7e 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -107,6 +107,7 @@ extern crate ethcore_stratum; extern crate ethabi; extern crate hardware_wallet; extern crate stats; +extern crate itertools; #[macro_use] extern crate log; diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index c417a79a1..62385d277 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -23,6 +23,7 @@ use snapshot::Error; use util::{U256, FixedHash, H256, Bytes, HashDB, SHA3_EMPTY, SHA3_NULL_RLP}; use util::trie::{TrieDB, Trie}; use rlp::{RlpStream, Stream, UntrustedRlp, View}; +use itertools::Itertools; use std::collections::HashSet; @@ -60,55 +61,53 @@ impl CodeState { } } -// walk the account's storage trie, returning an RLP item containing the -// account properties and the storage. -pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet) -> Result { +// walk the account's storage trie, returning a vector of RLP items containing the +// account properties and the storage. Each item contains at most `max_storage_items` +// storage records split according to snapshot format definition. +pub fn to_fat_rlps(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet, max_storage_items: usize) -> Result, Error> { if acc == &ACC_EMPTY { - return Ok(::rlp::NULL_RLP.to_vec()); + return Ok(vec![::rlp::NULL_RLP.to_vec()]); } let db = TrieDB::new(acct_db, &acc.storage_root)?; - let mut pairs = Vec::new(); + let chunks = db.iter()?.chunks(max_storage_items); + let pair_chunks = chunks.into_iter().map(|chunk| chunk.collect()); + pair_chunks.pad_using(1, |_| Vec::new(), ).map(|pairs| { + let mut stream = RlpStream::new_list(pairs.len()); - for item in db.iter()? { - let (k, v) = item?; - pairs.push((k, v)); - } + for r in pairs { + let (k, v) = r?; + stream.begin_list(2).append(&k).append(&&*v); + } - let mut stream = RlpStream::new_list(pairs.len()); + let pairs_rlp = stream.out(); - for (k, v) in pairs { - stream.begin_list(2).append(&k).append(&&*v); - } + let mut account_stream = RlpStream::new_list(5); + account_stream.append(&acc.nonce) + .append(&acc.balance); - let pairs_rlp = stream.out(); - - let mut account_stream = RlpStream::new_list(5); - account_stream.append(&acc.nonce) - .append(&acc.balance); - - // [has_code, code_hash]. - if acc.code_hash == SHA3_EMPTY { - account_stream.append(&CodeState::Empty.raw()).append_empty_data(); - } else if used_code.contains(&acc.code_hash) { - account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash); - } else { - match acct_db.get(&acc.code_hash) { - Some(c) => { - used_code.insert(acc.code_hash.clone()); - account_stream.append(&CodeState::Inline.raw()).append(&&*c); - } - None => { - warn!("code lookup failed during snapshot"); - account_stream.append(&false).append_empty_data(); + // [has_code, code_hash]. + if acc.code_hash == SHA3_EMPTY { + account_stream.append(&CodeState::Empty.raw()).append_empty_data(); + } else if used_code.contains(&acc.code_hash) { + account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash); + } else { + match acct_db.get(&acc.code_hash) { + Some(c) => { + used_code.insert(acc.code_hash.clone()); + account_stream.append(&CodeState::Inline.raw()).append(&&*c); + } + None => { + warn!("code lookup failed during snapshot"); + account_stream.append(&false).append_empty_data(); + } } } - } - account_stream.append_raw(&pairs_rlp, 1); - - Ok(account_stream.out()) + account_stream.append_raw(&pairs_rlp, 1); + Ok(account_stream.out()) + }).collect() } // decode a fat rlp, and rebuild the storage trie as we go. @@ -117,6 +116,7 @@ pub fn to_fat_rlp(acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashS pub fn from_fat_rlp( acct_db: &mut AccountDBMut, rlp: UntrustedRlp, + mut storage_root: H256, ) -> Result<(BasicAccount, Option), Error> { use util::{TrieDBMut, TrieMut}; @@ -148,10 +148,12 @@ pub fn from_fat_rlp( } }; - let mut storage_root = H256::zero(); - { - let mut storage_trie = TrieDBMut::new(acct_db, &mut storage_root); + let mut storage_trie = if storage_root.is_zero() { + TrieDBMut::new(acct_db, &mut storage_root) + } else { + TrieDBMut::from_existing(acct_db, &mut storage_root)? + }; let pairs = rlp.at(4)?; for pair_rlp in pairs.iter() { let k: Bytes = pair_rlp.val_at(0)?; @@ -184,7 +186,7 @@ mod tests { use std::collections::HashSet; - use super::{ACC_EMPTY, to_fat_rlp, from_fat_rlp}; + use super::{ACC_EMPTY, to_fat_rlps, from_fat_rlp}; #[test] fn encoding_basic() { @@ -201,9 +203,9 @@ mod tests { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp), account); - let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap(); - let fat_rlp = UntrustedRlp::new(&fat_rlp); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account); + let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap(); + let fat_rlp = UntrustedRlp::new(&fat_rlps[0]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account); } #[test] @@ -226,9 +228,40 @@ mod tests { let thin_rlp = ::rlp::encode(&account); assert_eq!(::rlp::decode::(&thin_rlp), account); - let fat_rlp = to_fat_rlp(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default()).unwrap(); - let fat_rlp = UntrustedRlp::new(&fat_rlp); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp).unwrap().0, account); + let fat_rlp = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), usize::max_value()).unwrap(); + let fat_rlp = UntrustedRlp::new(&fat_rlp[0]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, H256::zero()).unwrap().0, account); + } + + #[test] + fn encoding_storage_split() { + let mut db = get_temp_state_db(); + let addr = Address::random(); + + let account = { + let acct_db = AccountDBMut::new(db.as_hashdb_mut(), &addr); + let mut root = SHA3_NULL_RLP; + fill_storage(acct_db, &mut root, &mut H256::zero()); + BasicAccount { + nonce: 25.into(), + balance: 987654321.into(), + storage_root: root, + code_hash: SHA3_EMPTY, + } + }; + + let thin_rlp = ::rlp::encode(&account); + assert_eq!(::rlp::decode::(&thin_rlp), account); + + let fat_rlps = to_fat_rlps(&account, &AccountDB::new(db.as_hashdb(), &addr), &mut Default::default(), 100).unwrap(); + let mut root = SHA3_NULL_RLP; + let mut restored_account = None; + for rlp in fat_rlps { + let fat_rlp = UntrustedRlp::new(&rlp); + restored_account = Some(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr), fat_rlp, root).unwrap().0); + root = restored_account.as_ref().unwrap().storage_root.clone(); + } + assert_eq!(restored_account, Some(account)); } #[test] @@ -264,18 +297,18 @@ mod tests { let mut used_code = HashSet::new(); - let fat_rlp1 = to_fat_rlp(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code).unwrap(); - let fat_rlp2 = to_fat_rlp(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code).unwrap(); + let fat_rlp1 = to_fat_rlps(&account1, &AccountDB::new(db.as_hashdb(), &addr1), &mut used_code, usize::max_value()).unwrap(); + let fat_rlp2 = to_fat_rlps(&account2, &AccountDB::new(db.as_hashdb(), &addr2), &mut used_code, usize::max_value()).unwrap(); assert_eq!(used_code.len(), 1); - let fat_rlp1 = UntrustedRlp::new(&fat_rlp1); - let fat_rlp2 = UntrustedRlp::new(&fat_rlp2); + let fat_rlp1 = UntrustedRlp::new(&fat_rlp1[0]); + let fat_rlp2 = UntrustedRlp::new(&fat_rlp2[0]); - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2).unwrap(); + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr2), fat_rlp2, H256::zero()).unwrap(); assert!(maybe_code.is_none()); assert_eq!(acc, account2); - let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1).unwrap(); + let (acc, maybe_code) = from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &addr1), fat_rlp1, H256::zero()).unwrap(); assert_eq!(maybe_code, Some(b"this is definitely code".to_vec())); assert_eq!(acc, account1); } @@ -285,7 +318,7 @@ mod tests { let mut db = get_temp_state_db(); let mut used_code = HashSet::new(); - assert_eq!(to_fat_rlp(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code).unwrap(), ::rlp::NULL_RLP.to_vec()); - assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP)).unwrap(), (ACC_EMPTY, None)); + assert_eq!(to_fat_rlps(&ACC_EMPTY, &AccountDB::new(db.as_hashdb(), &Address::default()), &mut used_code, usize::max_value()).unwrap(), vec![::rlp::NULL_RLP.to_vec()]); + assert_eq!(from_fat_rlp(&mut AccountDBMut::new(db.as_hashdb_mut(), &Address::default()), UntrustedRlp::new(&::rlp::NULL_RLP), H256::zero()).unwrap(), (ACC_EMPTY, None)); } } diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 11491cd73..da8a9816b 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -53,6 +53,8 @@ pub enum Error { Decoder(DecoderError), /// Io error. Io(::std::io::Error), + /// Snapshot version is not supported. + VersionNotSupported(u64), } impl fmt::Display for Error { @@ -73,6 +75,7 @@ impl fmt::Display for Error { Error::Io(ref err) => err.fmt(f), Error::Decoder(ref err) => err.fmt(f), Error::Trie(ref err) => err.fmt(f), + Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver), } } } diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 362faf80b..d3d34ef52 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -31,6 +31,8 @@ use rlp::{self, Encodable, RlpStream, UntrustedRlp, Stream, View}; use super::ManifestData; +const SNAPSHOT_VERSION: u64 = 2; + /// Something which can write snapshots. /// Writing the same chunk multiple times will lead to implementation-defined /// behavior, and is not advised. @@ -120,8 +122,9 @@ impl SnapshotWriter for PackedWriter { fn finish(mut self, manifest: ManifestData) -> io::Result<()> { // we ignore the hashes fields of the manifest under the assumption that // they are consistent with ours. - let mut stream = RlpStream::new_list(5); + let mut stream = RlpStream::new_list(6); stream + .append(&SNAPSHOT_VERSION) .append(&self.state_hashes) .append(&self.block_hashes) .append(&manifest.state_root) @@ -223,7 +226,7 @@ impl PackedReader { /// Create a new `PackedReader` for the file at the given path. /// This will fail if any io errors are encountered or the file /// is not a valid packed snapshot. - pub fn new(path: &Path) -> Result, ::error::Error> { + pub fn new(path: &Path) -> Result, ::snapshot::error::Error> { let mut file = File::open(path)?; let file_len = file.metadata()?.len(); if file_len < 8 { @@ -257,15 +260,26 @@ impl PackedReader { let rlp = UntrustedRlp::new(&manifest_buf); - let state: Vec = rlp.val_at(0)?; - let blocks: Vec = rlp.val_at(1)?; + let (start, version) = if rlp.item_count() == 5 { + (0, 1) + } else { + (1, rlp.val_at(0)?) + }; + + if version > SNAPSHOT_VERSION { + return Err(::snapshot::error::Error::VersionNotSupported(version)); + } + + let state: Vec = rlp.val_at(0 + start)?; + let blocks: Vec = rlp.val_at(1 + start)?; let manifest = ManifestData { + version: version, state_hashes: state.iter().map(|c| c.0).collect(), block_hashes: blocks.iter().map(|c| c.0).collect(), - state_root: rlp.val_at(2)?, - block_number: rlp.val_at(3)?, - block_hash: rlp.val_at(4)?, + state_root: rlp.val_at(2 + start)?, + block_number: rlp.val_at(3 + start)?, + block_hash: rlp.val_at(4 + start)?, }; Ok(Some(PackedReader { @@ -348,7 +362,7 @@ mod tests { use util::sha3::Hashable; use snapshot::ManifestData; - use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader}; + use super::{SnapshotWriter, SnapshotReader, PackedWriter, PackedReader, LooseWriter, LooseReader, SNAPSHOT_VERSION}; const STATE_CHUNKS: &'static [&'static [u8]] = &[b"dog", b"cat", b"hello world", b"hi", b"notarealchunk"]; const BLOCK_CHUNKS: &'static [&'static [u8]] = &[b"hello!", b"goodbye!", b"abcdefg", b"hijklmnop", b"qrstuvwxy", b"and", b"z"]; @@ -374,6 +388,7 @@ mod tests { } let manifest = ManifestData { + version: SNAPSHOT_VERSION, state_hashes: state_hashes, block_hashes: block_hashes, state_root: b"notarealroot".sha3(), @@ -412,6 +427,7 @@ mod tests { } let manifest = ManifestData { + version: SNAPSHOT_VERSION, state_hashes: state_hashes, block_hashes: block_hashes, state_root: b"notarealroot".sha3(), @@ -428,4 +444,4 @@ mod tests { reader.chunk(hash.clone()).unwrap(); } } -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 53aa428b6..50246e008 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -56,6 +56,7 @@ pub use self::traits::SnapshotService; pub use self::watcher::Watcher; pub use types::snapshot_manifest::ManifestData; pub use types::restoration_status::RestorationStatus; +pub use types::basic_account::BasicAccount; pub mod io; pub mod service; @@ -82,6 +83,9 @@ mod traits { // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; +// Try to have chunks be around 4MB (before compression) +const MAX_STORAGE_ENTRIES_PER_ACCOUNT_RECORD: usize = 80_000; + // How many blocks to include in a snapshot, starting from the head of the chain. const SNAPSHOT_BLOCKS: u64 = 30000; @@ -147,6 +151,7 @@ pub fn take_snapshot( info!("produced {} state chunks and {} block chunks.", state_hashes.len(), block_hashes.len()); let manifest_data = ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: block_hashes, state_root: *state_root, @@ -300,14 +305,14 @@ impl<'a> StateChunker<'a> { // // If the buffer is greater than the desired chunk size, // this will write out the data to disk. - fn push(&mut self, account_hash: Bytes, data: Bytes) -> Result<(), Error> { + fn push(&mut self, account_hash: Bytes, data: Bytes, force_chunk: bool) -> Result<(), Error> { let pair = { let mut stream = RlpStream::new_list(2); stream.append(&account_hash).append_raw(&data, 1); stream.out() }; - if self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE { + if force_chunk || self.cur_size + pair.len() >= PREFERRED_CHUNK_SIZE { self.write_chunk()?; } @@ -372,8 +377,10 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex 0)?; + } } if chunker.cur_size != 0 { @@ -390,6 +397,7 @@ pub struct StateRebuilder { known_code: HashMap, // code hashes mapped to first account with this code. missing_code: HashMap>, // maps code hashes to lists of accounts missing that code. bloom: Bloom, + known_storage_roots: HashMap, // maps account hashes to last known storage root. Only filled for last account per chunk. } impl StateRebuilder { @@ -401,6 +409,7 @@ impl StateRebuilder { known_code: HashMap::new(), missing_code: HashMap::new(), bloom: StateDB::load_bloom(&*db), + known_storage_roots: HashMap::new(), } } @@ -418,6 +427,7 @@ impl StateRebuilder { rlp, &mut pairs, &self.known_code, + &mut self.known_storage_roots, flag )?; @@ -464,14 +474,18 @@ impl StateRebuilder { Ok(()) } - /// Check for accounts missing code. Once all chunks have been fed, there should - /// be none. - pub fn check_missing(self) -> Result<(), Error> { + /// Finalize the restoration. Check for accounts missing code and make a dummy + /// journal entry. + /// Once all chunks have been fed, there should be nothing missing. + pub fn finalize(mut self, era: u64, id: H256) -> Result<(), ::error::Error> { let missing = self.missing_code.keys().cloned().collect::>(); - match missing.is_empty() { - true => Ok(()), - false => Err(Error::MissingCode(missing)), - } + if !missing.is_empty() { return Err(Error::MissingCode(missing).into()) } + + let mut batch = self.db.backing().transaction(); + self.db.journal_under(&mut batch, era, &id)?; + self.db.backing().write_buffered(batch); + + Ok(()) } /// Get the state root of the rebuilder. @@ -492,10 +506,11 @@ fn rebuild_accounts( account_fat_rlps: UntrustedRlp, out_chunk: &mut [(H256, Bytes)], known_code: &HashMap, + known_storage_roots: &mut HashMap, abort_flag: &AtomicBool, ) -> Result { let mut status = RebuiltStatus::default(); - for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk) { + for (account_rlp, out) in account_fat_rlps.into_iter().zip(out_chunk.iter_mut()) { if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) } let hash: H256 = account_rlp.val_at(0)?; @@ -506,7 +521,8 @@ fn rebuild_accounts( // fill out the storage trie and code while decoding. let (acc, maybe_code) = { let mut acct_db = AccountDBMut::from_hash(db, hash); - account::from_fat_rlp(&mut acct_db, fat_rlp)? + let storage_root = known_storage_roots.get(&hash).cloned().unwrap_or(H256::zero()); + account::from_fat_rlp(&mut acct_db, fat_rlp, storage_root)? }; let code_hash = acc.code_hash.clone(); @@ -538,6 +554,12 @@ fn rebuild_accounts( *out = (hash, thin_rlp); } + if let Some(&(ref hash, ref rlp)) = out_chunk.iter().last() { + known_storage_roots.insert(*hash, ::rlp::decode::(rlp).storage_root); + } + if let Some(&(ref hash, ref rlp)) = out_chunk.iter().next() { + known_storage_roots.insert(*hash, ::rlp::decode::(rlp).storage_root); + } Ok(status) } diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index a354f3fd3..06e659bc1 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -166,7 +166,7 @@ impl Restoration { } // check for missing code. - self.state.check_missing()?; + self.state.finalize(self.manifest.block_number, self.manifest.block_hash)?; // connect out-of-order chunks and verify chain integrity. self.blocks.finalize(self.canonical_hashes)?; @@ -656,6 +656,7 @@ mod tests { assert_eq!(service.status(), RestorationStatus::Inactive); let manifest = ManifestData { + version: 2, state_hashes: vec![], block_hashes: vec![], state_root: Default::default(), diff --git a/ethcore/src/snapshot/tests/blocks.rs b/ethcore/src/snapshot/tests/blocks.rs index 89a01fbd7..f7c7fb97d 100644 --- a/ethcore/src/snapshot/tests/blocks.rs +++ b/ethcore/src/snapshot/tests/blocks.rs @@ -63,6 +63,7 @@ fn chunk_and_restore(amount: u64) { let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); let block_hashes = chunk_blocks(&bc, best_hash, &writer, &Progress::default()).unwrap(); let manifest = ::snapshot::ManifestData { + version: 2, state_hashes: Vec::new(), block_hashes: block_hashes, state_root: ::util::sha3::SHA3_NULL_RLP, @@ -125,6 +126,7 @@ fn checks_flag() { let chain = BlockChain::new(Default::default(), &genesis, db.clone()); let manifest = ::snapshot::ManifestData { + version: 2, state_hashes: Vec::new(), block_hashes: Vec::new(), state_root: ::util::sha3::SHA3_NULL_RLP, diff --git a/ethcore/src/snapshot/tests/mod.rs b/ethcore/src/snapshot/tests/mod.rs index e63cd6c7c..6530bb42a 100644 --- a/ethcore/src/snapshot/tests/mod.rs +++ b/ethcore/src/snapshot/tests/mod.rs @@ -27,6 +27,7 @@ use super::ManifestData; #[test] fn manifest_rlp() { let manifest = ManifestData { + version: 2, block_hashes: Vec::new(), state_hashes: Vec::new(), block_number: 1234567, @@ -35,4 +36,4 @@ fn manifest_rlp() { }; let raw = manifest.clone().into_rlp(); assert_eq!(ManifestData::from_rlp(&raw).unwrap(), manifest); -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/src/snapshot/tests/service.rs index 555ee665b..64a8407aa 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/src/snapshot/tests/service.rs @@ -122,6 +122,7 @@ fn guards_delete_folders() { path.push("restoration"); let manifest = ManifestData { + version: 2, state_hashes: vec![], block_hashes: vec![], block_number: 0, diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index 90c9e990f..8a74cc84a 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -58,10 +58,11 @@ fn snap_and_restore() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap(); writer.into_inner().finish(::snapshot::ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: Vec::new(), state_root: state_root, - block_number: 0, + block_number: 1000, block_hash: H256::default(), }).unwrap(); @@ -69,7 +70,7 @@ fn snap_and_restore() { db_path.push("db"); let db = { let new_db = Arc::new(Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap()); - let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::Archive); + let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent); let reader = PackedReader::new(&snap_file).unwrap().unwrap(); let flag = AtomicBool::new(true); @@ -82,12 +83,13 @@ fn snap_and_restore() { } assert_eq!(rebuilder.state_root(), state_root); - rebuilder.check_missing().unwrap(); + rebuilder.finalize(1000, H256::default()).unwrap(); new_db }; - let new_db = journaldb::new(db, Algorithm::Archive, ::db::COL_STATE); + let new_db = journaldb::new(db, Algorithm::OverlayRecent, ::db::COL_STATE); + assert_eq!(new_db.earliest_era(), Some(1000)); compare_dbs(&old_db, new_db.as_hashdb()); } @@ -120,10 +122,10 @@ fn get_code_from_prev_chunk() { let mut db = MemoryDB::new(); AccountDBMut::from_hash(&mut db, hash).insert(&code[..]); - let fat_rlp = account::to_fat_rlp(&acc, &AccountDB::from_hash(&db, hash), &mut used_code).unwrap(); + let fat_rlp = account::to_fat_rlps(&acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value()).unwrap(); let mut stream = RlpStream::new_list(1); - stream.begin_list(2).append(&hash).append_raw(&fat_rlp, 1); + stream.begin_list(2).append(&hash).append_raw(&fat_rlp[0], 1); stream.out() }; @@ -134,13 +136,18 @@ fn get_code_from_prev_chunk() { let db_cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS); let new_db = Arc::new(Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap()); - let mut rebuilder = StateRebuilder::new(new_db, Algorithm::Archive); - let flag = AtomicBool::new(true); + { + let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent); + let flag = AtomicBool::new(true); - rebuilder.feed(&chunk1, &flag).unwrap(); - rebuilder.feed(&chunk2, &flag).unwrap(); + rebuilder.feed(&chunk1, &flag).unwrap(); + rebuilder.feed(&chunk2, &flag).unwrap(); - rebuilder.check_missing().unwrap(); + rebuilder.finalize(1000, H256::random()).unwrap(); + } + + let state_db = journaldb::new(new_db, Algorithm::OverlayRecent, ::db::COL_STATE); + assert_eq!(state_db.earliest_era(), Some(1000)); } #[test] @@ -164,6 +171,7 @@ fn checks_flag() { let state_hashes = chunk_state(&old_db, &state_root, &writer, &Progress::default()).unwrap(); writer.into_inner().finish(::snapshot::ManifestData { + version: 2, state_hashes: state_hashes, block_hashes: Vec::new(), state_root: state_root, @@ -175,7 +183,7 @@ fn checks_flag() { db_path.push("db"); { let new_db = Arc::new(Database::open(&db_cfg, &db_path.to_string_lossy()).unwrap()); - let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::Archive); + let mut rebuilder = StateRebuilder::new(new_db.clone(), Algorithm::OverlayRecent); let reader = PackedReader::new(&snap_file).unwrap().unwrap(); let flag = AtomicBool::new(false); diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 3b7f84acf..8a1de711b 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -360,6 +360,10 @@ impl Spec { /// Account is marked with `reportBenign` it can be checked as disliked with "0xd8f2e0bf". /// Validator can be removed with `reportMalicious`. pub fn new_validator_contract() -> Self { load_bundled!("validator_contract") } + + /// Create a new Spec with BasicAuthority which uses multiple validator sets changing with height. + /// Account with secrets "0".sha3() is the validator for block 1 and with "1".sha3() onwards. + pub fn new_validator_multi() -> Self { load_bundled!("validator_multi") } } #[cfg(test)] diff --git a/ethcore/src/types/snapshot_manifest.rs b/ethcore/src/types/snapshot_manifest.rs index 24f56efeb..a4a68870d 100644 --- a/ethcore/src/types/snapshot_manifest.rs +++ b/ethcore/src/types/snapshot_manifest.rs @@ -24,6 +24,8 @@ use util::Bytes; #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "ipc", binary)] pub struct ManifestData { + /// Snapshot format version. + pub version: u64, /// List of state chunk hashes. pub state_hashes: Vec, /// List of block chunk hashes. @@ -39,7 +41,8 @@ pub struct ManifestData { impl ManifestData { /// Encode the manifest data to rlp. pub fn into_rlp(self) -> Bytes { - let mut stream = RlpStream::new_list(5); + let mut stream = RlpStream::new_list(6); + stream.append(&self.version); stream.append(&self.state_hashes); stream.append(&self.block_hashes); stream.append(&self.state_root); @@ -52,14 +55,20 @@ impl ManifestData { /// Try to restore manifest data from raw bytes, interpreted as RLP. pub fn from_rlp(raw: &[u8]) -> Result { let decoder = UntrustedRlp::new(raw); + let (start, version) = if decoder.item_count() == 5 { + (0, 1) + } else { + (1, decoder.val_at(0)?) + }; - let state_hashes: Vec = decoder.val_at(0)?; - let block_hashes: Vec = decoder.val_at(1)?; - let state_root: H256 = decoder.val_at(2)?; - let block_number: u64 = decoder.val_at(3)?; - let block_hash: H256 = decoder.val_at(4)?; + let state_hashes: Vec = decoder.val_at(start + 0)?; + let block_hashes: Vec = decoder.val_at(start + 1)?; + let state_root: H256 = decoder.val_at(start + 2)?; + let block_number: u64 = decoder.val_at(start + 3)?; + let block_hash: H256 = decoder.val_at(start + 4)?; Ok(ManifestData { + version: version, state_hashes: state_hashes, block_hashes: block_hashes, state_root: state_root, diff --git a/js/src/ui/Container/container.css b/js/src/ui/Container/container.css index d496931b9..3d66d2655 100644 --- a/js/src/ui/Container/container.css +++ b/js/src/ui/Container/container.css @@ -24,6 +24,7 @@ $transitionAll: all 0.75s cubic-bezier(0.23, 1, 0.32, 1); flex: 1; height: 100%; padding: 0em; + max-width: 100%; position: relative; /*transform: translateZ(0); transition: $transitionAll;*/ diff --git a/js/src/ui/SectionList/sectionList.css b/js/src/ui/SectionList/sectionList.css index 4613371ad..e81d8ce07 100644 --- a/js/src/ui/SectionList/sectionList.css +++ b/js/src/ui/SectionList/sectionList.css @@ -18,6 +18,7 @@ $transition: all 0.25s; $widthNormal: 33.33%; $widthExpanded: 42%; +$widthContracted: 29%; .section { position: relative; @@ -45,6 +46,7 @@ $widthExpanded: 42%; display: flex; flex: 0 1 $widthNormal; max-width: $widthNormal; + min-width: $widthContracted; opacity: 0.85; padding: 0.25em; diff --git a/json/src/spec/validator_set.rs b/json/src/spec/validator_set.rs index 080a36c50..f433caa03 100644 --- a/json/src/spec/validator_set.rs +++ b/json/src/spec/validator_set.rs @@ -16,6 +16,8 @@ //! Validator set deserialization. +use std::collections::BTreeMap; +use uint::Uint; use hash::Address; /// Different ways of specifying validators. @@ -30,6 +32,9 @@ pub enum ValidatorSet { /// Address of a contract that indicates the list of authorities and enables reporting of theor misbehaviour using transactions. #[serde(rename="contract")] Contract(Address), + /// A map of starting blocks for each validator set. + #[serde(rename="multi")] + Multi(BTreeMap), } #[cfg(test)] @@ -40,11 +45,17 @@ mod tests { #[test] fn validator_set_deserialization() { let s = r#"[{ - "list" : ["0xc6d9d2cd449a754c494264e1809c50e34d64562b"] + "list": ["0xc6d9d2cd449a754c494264e1809c50e34d64562b"] }, { - "safeContract" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b" + "safeContract": "0xc6d9d2cd449a754c494264e1809c50e34d64562b" }, { - "contract" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b" + "contract": "0xc6d9d2cd449a754c494264e1809c50e34d64562b" + }, { + "multi": { + "0": { "list": ["0xc6d9d2cd449a754c494264e1809c50e34d64562b"] }, + "10": { "list": ["0xd6d9d2cd449a754c494264e1809c50e34d64562b"] }, + "20": { "contract": "0xc6d9d2cd449a754c494264e1809c50e34d64562b" } + } }]"#; let _deserialized: Vec = serde_json::from_str(s).unwrap(); diff --git a/mac/Parity.pkgproj b/mac/Parity.pkgproj index 9245d1213..deedb7187 100755 --- a/mac/Parity.pkgproj +++ b/mac/Parity.pkgproj @@ -462,7 +462,7 @@ OVERWRITE_PERMISSIONS VERSION - 1.6.4 + 1.6.5 UUID 2DCD5B81-7BAF-4DA1-9251-6274B089FD36 diff --git a/mac/Parity/Info.plist b/mac/Parity/Info.plist index 7b7b733d2..56e623905 100644 --- a/mac/Parity/Info.plist +++ b/mac/Parity/Info.plist @@ -17,7 +17,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.6.4 + 1.6.5 CFBundleVersion 1 LSApplicationCategoryType diff --git a/nsis/installer.nsi b/nsis/installer.nsi index 7317dc529..75dd73dac 100644 --- a/nsis/installer.nsi +++ b/nsis/installer.nsi @@ -10,7 +10,7 @@ !define DESCRIPTION "Fast, light, robust Ethereum implementation" !define VERSIONMAJOR 1 !define VERSIONMINOR 6 -!define VERSIONBUILD 4 +!define VERSIONBUILD 5 !define ARGS "--warp" !define FIRST_START_ARGS "ui --warp --mode=passive" diff --git a/sync/src/chain.rs b/sync/src/chain.rs index ef35f0640..714b50dc3 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -158,6 +158,8 @@ pub const SNAPSHOT_SYNC_PACKET_COUNT: u8 = 0x16; const MAX_SNAPSHOT_CHUNKS_DOWNLOAD_AHEAD: usize = 3; +const MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION: u64 = 1; + const WAIT_PEERS_TIMEOUT_SEC: u64 = 5; const STATUS_TIMEOUT_SEC: u64 = 5; const HEADERS_TIMEOUT_SEC: u64 = 15; @@ -1023,12 +1025,18 @@ impl ChainSync { let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) { Err(e) => { trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e); - io.disconnect_peer(peer_id); + io.disable_peer(peer_id); self.continue_sync(io); return Ok(()); } Ok(manifest) => manifest, }; + if manifest.version < MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION { + trace!(target: "sync", "{}: Snapshot manifest version too low: {}", peer_id, manifest.version); + io.disable_peer(peer_id); + self.continue_sync(io); + return Ok(()); + } self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3()); io.snapshot_service().begin_restore(manifest); self.state = SyncState::SnapshotData; diff --git a/sync/src/snapshot.rs b/sync/src/snapshot.rs index c19860101..a585520d5 100644 --- a/sync/src/snapshot.rs +++ b/sync/src/snapshot.rs @@ -139,6 +139,7 @@ mod test { let state_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); let block_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); let manifest = ManifestData { + version: 2, state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), state_root: H256::new(), diff --git a/sync/src/tests/snapshot.rs b/sync/src/tests/snapshot.rs index 0affb0b1a..16114e216 100644 --- a/sync/src/tests/snapshot.rs +++ b/sync/src/tests/snapshot.rs @@ -49,6 +49,7 @@ impl TestSnapshotService { let state_chunks: Vec = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect(); let block_chunks: Vec = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect(); let manifest = ManifestData { + version: 2, state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), state_root: H256::new(), diff --git a/updater/src/updater.rs b/updater/src/updater.rs index a6c374e10..624731342 100644 --- a/updater/src/updater.rs +++ b/updater/src/updater.rs @@ -274,7 +274,7 @@ impl Updater { let running_latest = latest.track.version.hash == self.version_info().hash; let already_have_latest = s.installed.as_ref().or(s.ready.as_ref()).map_or(false, |t| *t == latest.track); - if self.update_policy.enable_downloading && !running_later && !running_latest && !already_have_latest { + if !s.disabled && self.update_policy.enable_downloading && !running_later && !running_latest && !already_have_latest { if let Some(b) = latest.track.binary { if s.fetching.is_none() { if self.updates_path(&Self::update_file_name(&latest.track.version)).exists() { diff --git a/util/Cargo.toml b/util/Cargo.toml index d48babd4e..17b762305 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -3,7 +3,7 @@ description = "Ethcore utility library" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore-util" -version = "1.6.4" +version = "1.6.5" authors = ["Parity Technologies "] build = "build.rs" diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index ed39905a2..247508bec 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -380,10 +380,7 @@ impl JournalDB for OverlayRecentDB { match rc { 0 => {} - 1 => { - if cfg!(debug_assertions) && self.backing.get(self.column, &key)?.is_some() { - return Err(BaseDataError::AlreadyExists(key).into()); - } + _ if rc > 0 => { batch.put(self.column, &key, &value) } -1 => { @@ -392,7 +389,7 @@ impl JournalDB for OverlayRecentDB { } batch.delete(self.column, &key) } - _ => panic!("Attempted to inject invalid state."), + _ => panic!("Attempted to inject invalid state ({})", rc), } }