diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf11cad67..e2cb77606 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,13 @@ test-linux: tags: - rust-stable +test-audit: + stage: test + script: + - scripts/gitlab/cargo-audit.sh + tags: + - rust-stable + build-linux: stage: build only: *releaseable_branches @@ -104,25 +111,18 @@ publish-awss3: tags: - shell -docs-jsonrpc: - stage: optional +publish-docs: + stage: publish only: - tags except: - nightly cache: {} script: - - scripts/gitlab/docs-jsonrpc.sh + - scripts/gitlab/publish-docs.sh tags: - shell -cargo-audit: - stage: optional - script: - - scripts/gitlab/cargo-audit.sh - tags: - - rust-stable - build-android: stage: optional image: parity/rust-android:gitlab-ci @@ -132,6 +132,7 @@ build-android: - scripts/gitlab/build-unix.sh tags: - rust-arm + allow_failure: true test-beta: stage: optional @@ -141,6 +142,7 @@ test-beta: - scripts/gitlab/test-all.sh beta tags: - rust-beta + allow_failure: true test-nightly: stage: optional @@ -150,3 +152,4 @@ test-nightly: - scripts/gitlab/test-all.sh nightly tags: - rust-nightly + allow_failure: true diff --git a/Cargo.lock b/Cargo.lock index eaa74650e..e5e6c31b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,7 +205,6 @@ name = "chainspec" version = "0.1.0" dependencies = [ "ethjson 0.1.0", - "serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -460,6 +459,15 @@ dependencies = [ "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "env_logger" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "env_logger" version = "0.5.13" @@ -483,10 +491,11 @@ dependencies = [ [[package]] name = "eth-secp256k1" version = "0.5.7" -source = "git+https://github.com/paritytech/rust-secp256k1#db81cfea59014b4d176f10f86ed52e1a130b6822" +source = "git+https://github.com/paritytech/rust-secp256k1#ccc06e7480148b723eb44ac56cf4d20eec380b6f" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -558,6 +567,7 @@ dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "common-types 0.1.0", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 6.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-contract 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1552,7 +1562,7 @@ dependencies = [ [[package]] name = "jsonrpc-core" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1564,7 +1574,7 @@ dependencies = [ [[package]] name = "jsonrpc-http-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1577,7 +1587,7 @@ dependencies = [ [[package]] name = "jsonrpc-ipc-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-server-utils 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1590,7 +1600,7 @@ dependencies = [ [[package]] name = "jsonrpc-macros" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-pubsub 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1600,7 +1610,7 @@ dependencies = [ [[package]] name = "jsonrpc-pubsub" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1610,7 +1620,7 @@ dependencies = [ [[package]] name = "jsonrpc-server-utils" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1626,7 +1636,7 @@ dependencies = [ [[package]] name = "jsonrpc-tcp-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-server-utils 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1638,7 +1648,7 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -2048,7 +2058,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2152,7 +2162,7 @@ version = "1.12.0" dependencies = [ "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", - "parity-ethereum 2.2.1", + "parity-ethereum 2.2.2", ] [[package]] @@ -2168,7 +2178,7 @@ dependencies = [ [[package]] name = "parity-ethereum" -version = "2.2.1" +version = "2.2.2" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2217,7 +2227,7 @@ dependencies = [ "parity-rpc-client 1.4.0", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.1", + "parity-version 2.2.2", "parity-whisper 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2372,7 +2382,7 @@ dependencies = [ "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.1", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2469,7 +2479,7 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.12.0", "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-version 2.2.1", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2479,7 +2489,7 @@ dependencies = [ [[package]] name = "parity-version" -version = "2.2.1" +version = "2.2.2" dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3107,14 +3117,6 @@ dependencies = [ "syn 0.15.11 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "serde_ignored" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "serde_json" version = "1.0.32" @@ -4083,6 +4085,7 @@ dependencies = [ "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" "checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" +"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" "checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" "checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02" "checksum eth-secp256k1 0.5.7 (git+https://github.com/paritytech/rust-secp256k1)" = "" @@ -4264,7 +4267,6 @@ dependencies = [ "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "15c141fc7027dd265a47c090bf864cf62b42c4d228bbcf4e51a0c9e2b0d3f7ef" "checksum serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "225de307c6302bec3898c51ca302fc94a7a1697ef0845fcee6448f33c032249c" -"checksum serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142" "checksum serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "43344e7ce05d0d8280c5940cabb4964bea626aa58b1ec0e8c73fa2a8512a38ce" "checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" "checksum sha1 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "171698ce4ec7cbb93babeb3190021b4d72e96ccb98e33d277ae4ea959d6f2d9e" diff --git a/Cargo.toml b/Cargo.toml index e0f197ecc..3af2dbab8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ description = "Parity Ethereum client" name = "parity-ethereum" # NOTE Make sure to update util/version/Cargo.toml as well -version = "2.2.1" +version = "2.2.2" license = "GPL-3.0" authors = ["Parity Technologies "] diff --git a/chainspec/Cargo.toml b/chainspec/Cargo.toml index 780fcdff4..8739ca7dc 100644 --- a/chainspec/Cargo.toml +++ b/chainspec/Cargo.toml @@ -6,4 +6,3 @@ authors = ["Marek Kotewicz "] [dependencies] ethjson = { path = "../json" } serde_json = "1.0" -serde_ignored = "0.0.4" diff --git a/chainspec/src/main.rs b/chainspec/src/main.rs index 708d74b50..43f9c8057 100644 --- a/chainspec/src/main.rs +++ b/chainspec/src/main.rs @@ -15,10 +15,8 @@ // along with Parity. If not, see . extern crate serde_json; -extern crate serde_ignored; extern crate ethjson; -use std::collections::BTreeSet; use std::{fs, env, process}; use ethjson::spec::Spec; @@ -41,24 +39,11 @@ fn main() { Err(_) => quit(&format!("{} could not be opened", path)), }; - let mut unused = BTreeSet::new(); - let mut deserializer = serde_json::Deserializer::from_reader(file); - - let spec: Result = serde_ignored::deserialize(&mut deserializer, |field| { - unused.insert(field.to_string()); - }); + let spec: Result = serde_json::from_reader(file); if let Err(err) = spec { quit(&format!("{} {}", path, err.to_string())); } - if !unused.is_empty() { - let err = unused.into_iter() - .map(|field| format!("{} unexpected field `{}`", path, field)) - .collect::>() - .join("\n"); - quit(&err); - } - println!("{} is valid", path); } diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 31e8da942..989c705d1 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -76,6 +76,7 @@ hardware-wallet = { path = "../hw" } fake-hardware-wallet = { path = "../util/fake-hardware-wallet" } [dev-dependencies] +env_logger = "0.4" tempdir = "0.3" trie-standardmap = "0.1" diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 590be038e..cec2c6994 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -28,7 +28,7 @@ use parking_lot::{Mutex, RwLock}; use provider::Provider; use request::{Request, NetworkRequests as Requests, Response}; use rlp::{RlpStream, Rlp}; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::ops::{BitOr, BitAnd, Not}; use std::sync::Arc; @@ -38,7 +38,7 @@ use std::time::{Duration, Instant}; use self::request_credits::{Credits, FlowParams}; use self::context::{Ctx, TickCtx}; use self::error::Punishment; -use self::load_timer::{LoadDistribution, NullStore}; +use self::load_timer::{LoadDistribution, NullStore, MOVING_SAMPLE_SIZE}; use self::request_set::RequestSet; use self::id_guard::IdGuard; @@ -70,6 +70,16 @@ const PROPAGATE_TIMEOUT_INTERVAL: Duration = Duration::from_secs(5); const RECALCULATE_COSTS_TIMEOUT: TimerToken = 3; const RECALCULATE_COSTS_INTERVAL: Duration = Duration::from_secs(60 * 60); +const STATISTICS_TIMEOUT: TimerToken = 4; +const STATISTICS_INTERVAL: Duration = Duration::from_secs(15); + +/// Maximum load share for the light server +pub const MAX_LIGHTSERV_LOAD: f64 = 0.5; + +/// Factor to multiply leecher count to cater for +/// extra sudden connections (should be >= 1.0) +pub const LEECHER_COUNT_FACTOR: f64 = 1.25; + // minimum interval between updates. const UPDATE_INTERVAL: Duration = Duration::from_millis(5000); @@ -256,18 +266,18 @@ pub trait Handler: Send + Sync { pub struct Config { /// How many stored seconds of credits peers should be able to accumulate. pub max_stored_seconds: u64, - /// How much of the total load capacity each peer should be allowed to take. - pub load_share: f64, + /// The network config median peers (used as default peer count) + pub median_peers: f64, } impl Default for Config { fn default() -> Self { - const LOAD_SHARE: f64 = 1.0 / 25.0; + const MEDIAN_PEERS: f64 = 25.0; const MAX_ACCUMULATED: u64 = 60 * 5; // only charge for 5 minutes. Config { max_stored_seconds: MAX_ACCUMULATED, - load_share: LOAD_SHARE, + median_peers: MEDIAN_PEERS, } } } @@ -335,6 +345,42 @@ mod id_guard { } } +/// Provides various statistics that could +/// be used to compute costs +pub struct Statistics { + /// Samples of peer count + peer_counts: VecDeque, +} + +impl Statistics { + /// Create a new Statistics instance + pub fn new() -> Self { + Statistics { + peer_counts: VecDeque::with_capacity(MOVING_SAMPLE_SIZE), + } + } + + /// Add a new peer_count sample + pub fn add_peer_count(&mut self, peer_count: usize) { + while self.peer_counts.len() >= MOVING_SAMPLE_SIZE { + self.peer_counts.pop_front(); + } + self.peer_counts.push_back(peer_count); + } + + /// Get the average peer count from previous samples. Is always >= 1.0 + pub fn avg_peer_count(&self) -> f64 { + let len = self.peer_counts.len(); + if len == 0 { + return 1.0; + } + let avg = self.peer_counts.iter() + .fold(0, |sum: u32, &v| sum.saturating_add(v as u32)) as f64 + / len as f64; + avg.max(1.0) + } +} + /// This is an implementation of the light ethereum network protocol, abstracted /// over a `Provider` of data and a p2p network. /// @@ -359,6 +405,7 @@ pub struct LightProtocol { req_id: AtomicUsize, sample_store: Box, load_distribution: LoadDistribution, + statistics: RwLock, } impl LightProtocol { @@ -369,9 +416,11 @@ impl LightProtocol { let genesis_hash = provider.chain_info().genesis_hash; let sample_store = params.sample_store.unwrap_or_else(|| Box::new(NullStore)); let load_distribution = LoadDistribution::load(&*sample_store); + // Default load share relative to median peers + let load_share = MAX_LIGHTSERV_LOAD / params.config.median_peers; let flow_params = FlowParams::from_request_times( |kind| load_distribution.expected_time(kind), - params.config.load_share, + load_share, Duration::from_secs(params.config.max_stored_seconds), ); @@ -389,6 +438,7 @@ impl LightProtocol { req_id: AtomicUsize::new(0), sample_store, load_distribution, + statistics: RwLock::new(Statistics::new()), } } @@ -408,6 +458,16 @@ impl LightProtocol { ) } + /// Get the number of active light peers downloading from the + /// node + pub fn leecher_count(&self) -> usize { + let credit_limit = *self.flow_params.read().limit(); + // Count the number of peers that used some credit + self.peers.read().iter() + .filter(|(_, p)| p.lock().local_credits.current() < credit_limit) + .count() + } + /// Make a request to a peer. /// /// Fails on: nonexistent peer, network error, peer not server, @@ -772,12 +832,16 @@ impl LightProtocol { fn begin_new_cost_period(&self, io: &IoContext) { self.load_distribution.end_period(&*self.sample_store); + let avg_peer_count = self.statistics.read().avg_peer_count(); + // Load share relative to average peer count +LEECHER_COUNT_FACTOR% + let load_share = MAX_LIGHTSERV_LOAD / (avg_peer_count * LEECHER_COUNT_FACTOR); let new_params = Arc::new(FlowParams::from_request_times( |kind| self.load_distribution.expected_time(kind), - self.config.load_share, + load_share, Duration::from_secs(self.config.max_stored_seconds), )); *self.flow_params.write() = new_params.clone(); + trace!(target: "pip", "New cost period: avg_peers={} ; cost_table:{:?}", avg_peer_count, new_params.cost_table()); let peers = self.peers.read(); let now = Instant::now(); @@ -797,6 +861,11 @@ impl LightProtocol { peer_info.awaiting_acknowledge = Some((now, new_params.clone())); } } + + fn tick_statistics(&self) { + let leecher_count = self.leecher_count(); + self.statistics.write().add_peer_count(leecher_count); + } } impl LightProtocol { @@ -1099,6 +1168,8 @@ impl NetworkProtocolHandler for LightProtocol { .expect("Error registering sync timer."); io.register_timer(RECALCULATE_COSTS_TIMEOUT, RECALCULATE_COSTS_INTERVAL) .expect("Error registering request timer interval token."); + io.register_timer(STATISTICS_TIMEOUT, STATISTICS_INTERVAL) + .expect("Error registering statistics timer."); } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { @@ -1119,6 +1190,7 @@ impl NetworkProtocolHandler for LightProtocol { TICK_TIMEOUT => self.tick_handlers(&io), PROPAGATE_TIMEOUT => self.propagate_transactions(&io), RECALCULATE_COSTS_TIMEOUT => self.begin_new_cost_period(&io), + STATISTICS_TIMEOUT => self.tick_statistics(), _ => warn!(target: "pip", "received timeout on unknown token {}", timer), } } diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index d1939015c..791315f5f 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -22,9 +22,10 @@ use ethcore::client::{EachBlockWith, TestBlockChainClient}; use ethcore::encoded; use ethcore::ids::BlockId; use ethereum_types::{H256, U256, Address}; -use net::{LightProtocol, Params, packet, Peer}; +use net::{LightProtocol, Params, packet, Peer, Statistics}; use net::context::IoContext; use net::status::{Capabilities, Status}; +use net::load_timer::MOVING_SAMPLE_SIZE; use network::{PeerId, NodeId}; use provider::Provider; use request; @@ -780,3 +781,34 @@ fn get_transaction_index() { let expected = Expect::Respond(packet::RESPONSE, response); proto.handle_packet(&expected, 1, packet::REQUEST, &request_body); } + +#[test] +fn sync_statistics() { + let mut stats = Statistics::new(); + + // Empty set should return 1.0 + assert_eq!(stats.avg_peer_count(), 1.0); + + // Average < 1.0 should return 1.0 + stats.add_peer_count(0); + assert_eq!(stats.avg_peer_count(), 1.0); + + stats = Statistics::new(); + + const N: f64 = 50.0; + + for i in 1..(N as usize + 1) { + stats.add_peer_count(i); + } + + // Compute the average for the sum 1..N + assert_eq!(stats.avg_peer_count(), N * (N + 1.0) / 2.0 / N); + + for _ in 1..(MOVING_SAMPLE_SIZE + 1) { + stats.add_peer_count(40); + } + + // Test that it returns the average of the last + // `MOVING_SAMPLE_SIZE` values + assert_eq!(stats.avg_peer_count(), 40.0); +} diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 723446206..bfd15dadc 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -176,8 +176,8 @@ impl Provider for T { } fn block_receipts(&self, req: request::CompleteReceiptsRequest) -> Option { - BlockChainClient::encoded_block_receipts(self, &req.hash) - .map(|x| ::request::ReceiptsResponse { receipts: ::rlp::decode_list(&x) }) + BlockChainClient::block_receipts(self, &req.hash) + .map(|x| ::request::ReceiptsResponse { receipts: x.receipts }) } fn account_proof(&self, req: request::CompleteAccountRequest) -> Option { diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index a6bbc4e10..77429a4e5 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -106,7 +106,13 @@ impl ClientService { info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name())); let pruning = config.pruning; - let client = Client::new(config, &spec, blockchain_db.clone(), miner.clone(), io_service.channel())?; + let client = Client::new( + config, + &spec, + blockchain_db.clone(), + miner.clone(), + io_service.channel(), + )?; miner.set_io_channel(io_service.channel()); miner.set_in_chain_checker(&client.clone()); @@ -117,7 +123,7 @@ impl ClientService { pruning: pruning, channel: io_service.channel(), snapshot_root: snapshot_path.into(), - db_restore: client.clone(), + client: client.clone(), }; let snapshot = Arc::new(SnapshotService::new(snapshot_params)?); diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 5785aa103..4286dc414 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -229,6 +229,7 @@ pub struct BlockChain { cache_man: Mutex>, + pending_best_ancient_block: RwLock>>, pending_best_block: RwLock>, pending_block_hashes: RwLock>, pending_block_details: RwLock>, @@ -538,6 +539,7 @@ impl BlockChain { block_receipts: RwLock::new(HashMap::new()), db: db.clone(), cache_man: Mutex::new(cache_man), + pending_best_ancient_block: RwLock::new(None), pending_best_block: RwLock::new(None), pending_block_hashes: RwLock::new(HashMap::new()), pending_block_details: RwLock::new(HashMap::new()), @@ -808,18 +810,7 @@ impl BlockChain { }, is_best); if is_ancient { - let mut best_ancient_block = self.best_ancient_block.write(); - let ancient_number = best_ancient_block.as_ref().map_or(0, |b| b.number); - if self.block_hash(block_number + 1).is_some() { - batch.delete(db::COL_EXTRA, b"ancient"); - *best_ancient_block = None; - } else if block_number > ancient_number { - batch.put(db::COL_EXTRA, b"ancient", &hash); - *best_ancient_block = Some(BestAncientBlock { - hash: hash, - number: block_number, - }); - } + self.set_best_ancient_block(block_number, &hash, batch); } false @@ -860,6 +851,84 @@ impl BlockChain { } } + /// Update the best ancient block to the given hash, after checking that + /// it's directly linked to the currently known best ancient block + pub fn update_best_ancient_block(&self, hash: &H256) { + // Get the block view of the next ancient block (it must + // be in DB at this point) + let block_view = match self.block(hash) { + Some(v) => v, + None => return, + }; + + // So that `best_ancient_block` gets unlocked before calling + // `set_best_ancient_block` + { + // Get the target hash ; if there are no ancient block, + // it means that the chain is already fully linked + // Release the `best_ancient_block` RwLock + let target_hash = { + let best_ancient_block = self.best_ancient_block.read(); + let cur_ancient_block = match *best_ancient_block { + Some(ref b) => b, + None => return, + }; + + // Ensure that the new best ancient block is after the current one + if block_view.number() <= cur_ancient_block.number { + return; + } + + cur_ancient_block.hash.clone() + }; + + let mut block_hash = *hash; + let mut is_linked = false; + + loop { + if block_hash == target_hash { + is_linked = true; + break; + } + + match self.block_details(&block_hash) { + Some(block_details) => { + block_hash = block_details.parent; + }, + None => break, + } + } + + if !is_linked { + trace!(target: "blockchain", "The given block {:x} is not linked to the known ancient block {:x}", hash, target_hash); + return; + } + } + + let mut batch = self.db.key_value().transaction(); + self.set_best_ancient_block(block_view.number(), hash, &mut batch); + self.db.key_value().write(batch).expect("Low level database error."); + } + + /// Set the best ancient block with the given value: private method + /// `best_ancient_block` must not be locked, otherwise a DeadLock would occur + fn set_best_ancient_block(&self, block_number: BlockNumber, block_hash: &H256, batch: &mut DBTransaction) { + let mut pending_best_ancient_block = self.pending_best_ancient_block.write(); + let ancient_number = self.best_ancient_block.read().as_ref().map_or(0, |b| b.number); + if self.block_hash(block_number + 1).is_some() { + trace!(target: "blockchain", "The two ends of the chain have met."); + batch.delete(db::COL_EXTRA, b"ancient"); + *pending_best_ancient_block = Some(None); + } else if block_number > ancient_number { + trace!(target: "blockchain", "Updating the best ancient block to {}.", block_number); + batch.put(db::COL_EXTRA, b"ancient", &block_hash); + *pending_best_ancient_block = Some(Some(BestAncientBlock { + hash: *block_hash, + number: block_number, + })); + } + } + /// Insert an epoch transition. Provide an epoch number being transitioned to /// and epoch transition object. /// @@ -1112,15 +1181,21 @@ impl BlockChain { /// Apply pending insertion updates pub fn commit(&self) { + let mut pending_best_ancient_block = self.pending_best_ancient_block.write(); let mut pending_best_block = self.pending_best_block.write(); let mut pending_write_hashes = self.pending_block_hashes.write(); let mut pending_block_details = self.pending_block_details.write(); let mut pending_write_txs = self.pending_transaction_addresses.write(); let mut best_block = self.best_block.write(); + let mut best_ancient_block = self.best_ancient_block.write(); let mut write_block_details = self.block_details.write(); let mut write_hashes = self.block_hashes.write(); let mut write_txs = self.transaction_addresses.write(); + // update best ancient block + if let Some(block_option) = pending_best_ancient_block.take() { + *best_ancient_block = block_option; + } // update best block if let Some(block) = pending_best_block.take() { *best_block = block; diff --git a/ethcore/src/client/chain_notify.rs b/ethcore/src/client/chain_notify.rs index ebfe7bdef..3d576ae12 100644 --- a/ethcore/src/client/chain_notify.rs +++ b/ethcore/src/client/chain_notify.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use bytes::Bytes; -use ethereum_types::H256; +use ethereum_types::{H256, U256}; use transaction::UnverifiedTransaction; use blockchain::ImportRoute; use std::time::Duration; @@ -141,7 +141,15 @@ pub trait ChainNotify : Send + Sync { } /// fires when chain broadcasts a message - fn broadcast(&self, _message_type: ChainMessageType) {} + fn broadcast(&self, _message_type: ChainMessageType) { + // does nothing by default + } + + /// fires when new block is about to be imported + /// implementations should be light + fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { + // does nothing by default + } /// fires when new transactions are received from a peer fn transactions_received(&self, diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index ff09f7c72..00dc9cab5 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -32,7 +32,7 @@ use kvdb::{DBValue, KeyValueDB, DBTransaction}; // other use ethereum_types::{H256, Address, U256}; use block::{IsBlock, LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock}; -use blockchain::{BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert}; +use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert}; use client::ancient_import::AncientVerifier; use client::{ Nonce, Balance, ChainInfo, BlockInfo, CallContract, TransactionInfo, @@ -66,7 +66,7 @@ use ethcore_miner::pool::VerifiedTransaction; use parking_lot::{Mutex, RwLock}; use rand::OsRng; use receipt::{Receipt, LocalizedReceipt}; -use snapshot::{self, io as snapshot_io}; +use snapshot::{self, io as snapshot_io, SnapshotClient}; use spec::Spec; use state_db::StateDB; use state::{self, State}; @@ -881,7 +881,7 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { self.importer.block_queue.flush(); - while !self.importer.block_queue.queue_info().is_empty() { + while !self.importer.block_queue.is_empty() { self.import_verified_blocks(); } } @@ -1005,6 +1005,16 @@ impl Client { self.importer.miner.clone() } + #[cfg(test)] + pub fn state_db(&self) -> ::parking_lot::RwLockReadGuard { + self.state_db.read() + } + + #[cfg(test)] + pub fn chain(&self) -> Arc { + self.chain.read().clone() + } + /// Replace io channel. Useful for testing. pub fn set_io_channel(&self, io_channel: IoChannel) { *self.io_channel.write() = io_channel; @@ -1413,8 +1423,21 @@ impl ImportBlock for Client { bail!(EthcoreErrorKind::Block(BlockError::UnknownParent(unverified.parent_hash()))); } + let raw = if self.importer.block_queue.is_empty() { + Some(( + unverified.bytes.clone(), + unverified.header.hash(), + *unverified.header.difficulty(), + )) + } else { None }; + match self.importer.block_queue.import(unverified) { - Ok(res) => Ok(res), + Ok(hash) => { + if let Some((raw, hash, difficulty)) = raw { + self.notify(move |n| n.block_pre_import(&raw, &hash, &difficulty)); + } + Ok(hash) + }, // we only care about block errors (not import errors) Err((block, EthcoreError(EthcoreErrorKind::Block(err), _))) => { self.importer.bad_blocks.report(block.bytes, format!("{:?}", err)); @@ -1817,7 +1840,7 @@ impl BlockChainClient for Client { Some(receipt) } - fn block_receipts(&self, id: BlockId) -> Option> { + fn localized_block_receipts(&self, id: BlockId) -> Option> { let hash = self.block_hash(id)?; let chain = self.chain.read(); @@ -1860,14 +1883,18 @@ impl BlockChainClient for Client { self.state_db.read().journal_db().state(hash) } - fn encoded_block_receipts(&self, hash: &H256) -> Option { - self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts)) + fn block_receipts(&self, hash: &H256) -> Option { + self.chain.read().block_receipts(hash) } fn queue_info(&self) -> BlockQueueInfo { self.importer.block_queue.queue_info() } + fn is_queue_empty(&self) -> bool { + self.importer.block_queue.is_empty() + } + fn clear_queue(&self) { self.importer.block_queue.clear(); } @@ -2277,26 +2304,37 @@ impl ScheduleInfo for Client { impl ImportSealedBlock for Client { fn import_sealed_block(&self, block: SealedBlock) -> EthcoreResult { - let h = block.header().hash(); let start = Instant::now(); + let raw = block.rlp_bytes(); + let header = block.header().clone(); + let hash = header.hash(); + self.notify(|n| n.block_pre_import(&raw, &hash, header.difficulty())); + let route = { + // Do a super duper basic verification to detect potential bugs + if let Err(e) = self.engine.verify_block_basic(&header) { + self.importer.bad_blocks.report( + block.rlp_bytes(), + format!("Detected an issue with locally sealed block: {}", e), + ); + return Err(e.into()); + } + // scope for self.import_lock let _import_lock = self.importer.import_lock.lock(); trace_time!("import_sealed_block"); - let number = block.header().number(); let block_data = block.rlp_bytes(); - let header = block.header().clone(); let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self); - trace!(target: "client", "Imported sealed block #{} ({})", number, h); + trace!(target: "client", "Imported sealed block #{} ({})", header.number(), hash); self.state_db.write().sync_cache(&route.enacted, &route.retracted, false); route }; let route = ChainRoute::from([route].as_ref()); self.importer.miner.chain_new_blocks( self, - &[h.clone()], + &[hash], &[], route.enacted(), route.retracted(), @@ -2304,16 +2342,16 @@ impl ImportSealedBlock for Client { ); self.notify(|notify| { notify.new_blocks( - vec![h.clone()], + vec![hash], vec![], route.clone(), - vec![h.clone()], + vec![hash], vec![], start.elapsed(), ); }); self.db.read().key_value().flush().expect("DB flush failed."); - Ok(h) + Ok(hash) } } @@ -2406,6 +2444,8 @@ impl ProvingBlockChainClient for Client { } } +impl SnapshotClient for Client {} + impl Drop for Client { fn drop(&mut self) { self.engine.stop(); @@ -2504,7 +2544,7 @@ mod tests { use test_helpers::{generate_dummy_client_with_data}; let client = generate_dummy_client_with_data(2, 2, &[1.into(), 1.into()]); - let receipts = client.block_receipts(BlockId::Latest).unwrap(); + let receipts = client.localized_block_receipts(BlockId::Latest).unwrap(); assert_eq!(receipts.len(), 2); assert_eq!(receipts[0].transaction_index, 0); diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index c3d8c127b..3c422c49a 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -686,7 +686,7 @@ impl BlockChainClient for TestBlockChainClient { self.receipts.read().get(&id).cloned() } - fn block_receipts(&self, _id: BlockId) -> Option> { + fn localized_block_receipts(&self, _id: BlockId) -> Option> { Some(self.receipts.read().values().cloned().collect()) } @@ -789,16 +789,14 @@ impl BlockChainClient for TestBlockChainClient { None } - fn encoded_block_receipts(&self, hash: &H256) -> Option { + fn block_receipts(&self, hash: &H256) -> Option { // starts with 'f' ? if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { let receipt = BlockReceipts::new(vec![Receipt::new( TransactionOutcome::StateRoot(H256::zero()), U256::zero(), vec![])]); - let mut rlp = RlpStream::new(); - rlp.append(&receipt); - return Some(rlp.out()); + return Some(receipt); } None } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index ebe70dcd8..55d527013 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use itertools::Itertools; use block::{OpenBlock, SealedBlock, ClosedBlock}; -use blockchain::TreeRoute; +use blockchain::{BlockReceipts, TreeRoute}; use client::Mode; use encoded; use vm::LastHashes; @@ -282,7 +282,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra fn transaction_receipt(&self, id: TransactionId) -> Option; /// Get localized receipts for all transaction in given block. - fn block_receipts(&self, id: BlockId) -> Option>; + fn localized_block_receipts(&self, id: BlockId) -> Option>; /// Get a tree route between `from` and `to`. /// See `BlockChain::tree_route`. @@ -294,12 +294,17 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra /// Get latest state node fn state_data(&self, hash: &H256) -> Option; - /// Get raw block receipts data by block header hash. - fn encoded_block_receipts(&self, hash: &H256) -> Option; + /// Get block receipts data by block header hash. + fn block_receipts(&self, hash: &H256) -> Option; /// Get block queue information. fn queue_info(&self) -> BlockQueueInfo; + /// Returns true if block queue is empty. + fn is_queue_empty(&self) -> bool { + self.queue_info().is_empty() + } + /// Clear block queue and abort all import activity. fn clear_queue(&self); diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 1a54ccec1..bf404b476 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -16,8 +16,8 @@ //! A blockchain engine that supports a non-instant BFT proof-of-authority. -use std::collections::{BTreeMap, HashSet}; -use std::fmt; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::{cmp, fmt}; use std::iter::FromIterator; use std::ops::Deref; use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; @@ -123,10 +123,10 @@ struct Step { } impl Step { - fn load(&self) -> usize { self.inner.load(AtomicOrdering::SeqCst) } + fn load(&self) -> u64 { self.inner.load(AtomicOrdering::SeqCst) as u64 } fn duration_remaining(&self) -> Duration { let now = unix_now(); - let expected_seconds = (self.load() as u64) + let expected_seconds = self.load() .checked_add(1) .and_then(|ctr| ctr.checked_mul(self.duration as u64)) .map(Duration::from_secs); @@ -162,8 +162,8 @@ impl Step { } } - fn check_future(&self, given: usize) -> Result<(), Option>> { - const REJECTED_STEP_DRIFT: usize = 4; + fn check_future(&self, given: u64) -> Result<(), Option>> { + const REJECTED_STEP_DRIFT: u64 = 4; // Verify if the step is correct. if given <= self.load() { @@ -182,8 +182,8 @@ impl Step { let d = self.duration as u64; Err(Some(OutOfBounds { min: None, - max: Some(d * current as u64), - found: d * given as u64, + max: Some(d * current), + found: d * given, })) } else { Ok(()) @@ -192,8 +192,8 @@ impl Step { } // Chain scoring: total weight is sqrt(U256::max_value())*height - step -fn calculate_score(parent_step: U256, current_step: U256, current_empty_steps: U256) -> U256 { - U256::from(U128::max_value()) + parent_step - current_step + current_empty_steps +fn calculate_score(parent_step: u64, current_step: u64, current_empty_steps: usize) -> U256 { + U256::from(U128::max_value()) + U256::from(parent_step) - U256::from(current_step) + U256::from(current_empty_steps) } struct EpochManager { @@ -284,13 +284,26 @@ impl EpochManager { /// A message broadcast by authorities when it's their turn to seal a block but there are no /// transactions. Other authorities accumulate these messages and later include them in the seal as /// proof. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] struct EmptyStep { signature: H520, - step: usize, + step: u64, parent_hash: H256, } +impl PartialOrd for EmptyStep { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for EmptyStep { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.step.cmp(&other.step) + .then_with(|| self.parent_hash.cmp(&other.parent_hash)) + .then_with(|| self.signature.cmp(&other.signature)) + } +} + impl EmptyStep { fn from_sealed(sealed_empty_step: SealedEmptyStep, parent_hash: &H256) -> EmptyStep { let signature = sealed_empty_step.signature; @@ -353,7 +366,7 @@ pub fn empty_step_full_rlp(signature: &H520, empty_step_rlp: &[u8]) -> Vec { s.out() } -pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec { +pub fn empty_step_rlp(step: u64, parent_hash: &H256) -> Vec { let mut s = RlpStream::new_list(2); s.append(&step).append(parent_hash); s.out() @@ -365,7 +378,7 @@ pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec { /// empty message is included. struct SealedEmptyStep { signature: H520, - step: usize, + step: u64, } impl Encodable for SealedEmptyStep { @@ -399,7 +412,7 @@ pub struct AuthorityRound { validators: Box, validate_score_transition: u64, validate_step_transition: u64, - empty_steps: Mutex>, + empty_steps: Mutex>, epoch_manager: Mutex, immediate_transitions: bool, block_reward: U256, @@ -494,7 +507,7 @@ fn header_expected_seal_fields(header: &Header, empty_steps_transition: u64) -> } } -fn header_step(header: &Header, empty_steps_transition: u64) -> Result { +fn header_step(header: &Header, empty_steps_transition: u64) -> Result { let expected_seal_fields = header_expected_seal_fields(header, empty_steps_transition); Rlp::new(&header.seal().get(0).expect( &format!("was either checked with verify_block_basic or is genesis; has {} fields; qed (Make sure the spec file has a correct genesis seal)", expected_seal_fields))).as_val() @@ -533,17 +546,17 @@ fn header_empty_steps_signers(header: &Header, empty_steps_transition: u64) -> R } } -fn step_proposer(validators: &ValidatorSet, bh: &H256, step: usize) -> Address { - let proposer = validators.get(bh, step); +fn step_proposer(validators: &ValidatorSet, bh: &H256, step: u64) -> Address { + let proposer = validators.get(bh, step as usize); trace!(target: "engine", "Fetched proposer for step {}: {}", step, proposer); proposer } -fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: usize, address: &Address) -> bool { +fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: u64, address: &Address) -> bool { step_proposer(validators, bh, step) == *address } -fn verify_timestamp(step: &Step, header_step: usize) -> Result<(), BlockError> { +fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> { match step.check_future(header_step) { Err(None) => { trace!(target: "engine", "verify_timestamp: block from the future"); @@ -564,7 +577,7 @@ fn verify_external(header: &Header, validators: &ValidatorSet, empty_steps_trans let header_step = header_step(header, empty_steps_transition)?; let proposer_signature = header_signature(header, empty_steps_transition)?; - let correct_proposer = validators.get(header.parent_hash(), header_step); + let correct_proposer = validators.get(header.parent_hash(), header_step as usize); let is_invalid_proposer = *header.author() != correct_proposer || { let empty_steps_rlp = if header.number() >= empty_steps_transition { Some(header_empty_steps_raw(header)) @@ -634,13 +647,13 @@ impl AuthorityRound { panic!("authority_round: step duration can't be zero") } let should_timeout = our_params.start_step.is_none(); - let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))) as usize; + let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))); let engine = Arc::new( AuthorityRound { transition_service: IoService::<()>::start()?, step: Arc::new(PermissionedStep { inner: Step { - inner: AtomicUsize::new(initial_step), + inner: AtomicUsize::new(initial_step as usize), calibrate: our_params.start_step.is_none(), duration: our_params.step_duration, }, @@ -651,7 +664,7 @@ impl AuthorityRound { validators: our_params.validators, validate_score_transition: our_params.validate_score_transition, validate_step_transition: our_params.validate_step_transition, - empty_steps: Mutex::new(Vec::new()), + empty_steps: Default::default(), epoch_manager: Mutex::new(EpochManager::blank()), immediate_transitions: our_params.immediate_transitions, block_reward: our_params.block_reward, @@ -699,22 +712,41 @@ impl AuthorityRound { }) } - fn empty_steps(&self, from_step: U256, to_step: U256, parent_hash: H256) -> Vec { - self.empty_steps.lock().iter().filter(|e| { - U256::from(e.step) > from_step && - U256::from(e.step) < to_step && - e.parent_hash == parent_hash - }).cloned().collect() + fn empty_steps(&self, from_step: u64, to_step: u64, parent_hash: H256) -> Vec { + let from = EmptyStep { + step: from_step + 1, + parent_hash, + signature: Default::default(), + }; + let to = EmptyStep { + step: to_step, + parent_hash: Default::default(), + signature: Default::default(), + }; + + if from >= to { + return vec![]; + } + + self.empty_steps.lock() + .range(from..to) + .filter(|e| e.parent_hash == parent_hash) + .cloned() + .collect() } - fn clear_empty_steps(&self, step: U256) { + fn clear_empty_steps(&self, step: u64) { // clear old `empty_steps` messages - self.empty_steps.lock().retain(|e| U256::from(e.step) > step); + let mut empty_steps = self.empty_steps.lock(); + *empty_steps = empty_steps.split_off(&EmptyStep { + step: step + 1, + parent_hash: Default::default(), + signature: Default::default(), + }); } fn handle_empty_step_message(&self, empty_step: EmptyStep) { - let mut empty_steps = self.empty_steps.lock(); - empty_steps.push(empty_step); + self.empty_steps.lock().insert(empty_step); } fn generate_empty_step(&self, parent_hash: &H256) { @@ -744,7 +776,7 @@ impl AuthorityRound { } } - fn report_skipped(&self, header: &Header, current_step: usize, parent_step: usize, validators: &ValidatorSet, set_number: u64) { + fn report_skipped(&self, header: &Header, current_step: u64, parent_step: u64, validators: &ValidatorSet, set_number: u64) { // we're building on top of the genesis block so don't report any skipped steps if header.number() == 1 { return; @@ -937,12 +969,12 @@ impl Engine for AuthorityRound { let current_step = self.step.inner.load(); let current_empty_steps_len = if header.number() >= self.empty_steps_transition { - self.empty_steps(parent_step.into(), current_step.into(), parent.hash()).len() + self.empty_steps(parent_step, current_step, parent.hash()).len() } else { 0 }; - let score = calculate_score(parent_step.into(), current_step.into(), current_empty_steps_len.into()); + let score = calculate_score(parent_step, current_step, current_empty_steps_len); header.set_difficulty(score); } @@ -986,8 +1018,8 @@ impl Engine for AuthorityRound { } let header = block.header(); - let parent_step: U256 = header_step(parent, self.empty_steps_transition) - .expect("Header has been verified; qed").into(); + let parent_step = header_step(parent, self.empty_steps_transition) + .expect("Header has been verified; qed"); let step = self.step.inner.load(); @@ -1022,7 +1054,7 @@ impl Engine for AuthorityRound { if is_step_proposer(&*validators, header.parent_hash(), step, header.author()) { // this is guarded against by `can_propose` unless the block was signed // on the same step (implies same key) and on a different node. - if parent_step == step.into() { + if parent_step == step { warn!("Attempted to seal block on the same step as parent. Is this authority sealing with more than one node?"); return Seal::None; } @@ -1034,7 +1066,10 @@ impl Engine for AuthorityRound { block.transactions().is_empty() && empty_steps.len() < self.maximum_empty_steps { - self.generate_empty_step(header.parent_hash()); + if self.step.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) { + self.generate_empty_step(header.parent_hash()); + } + return Seal::None; } @@ -1058,7 +1093,7 @@ impl Engine for AuthorityRound { // report any skipped primaries between the parent block and // the block we're sealing, unless we have empty steps enabled if header.number() < self.empty_steps_transition { - self.report_skipped(header, step, u64::from(parent_step) as usize, &*validators, set_number); + self.report_skipped(header, step, parent_step, &*validators, set_number); } let mut fields = vec![ @@ -1593,12 +1628,12 @@ mod tests { // Two validators. // Spec starts with step 2. - header.set_difficulty(calculate_score(U256::from(0), U256::from(2), U256::zero())); + header.set_difficulty(calculate_score(0, 2, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&2usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); assert!(engine.verify_block_external(&header).is_err()); - header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero())); + header.set_difficulty(calculate_score(0, 1, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); @@ -1622,7 +1657,7 @@ mod tests { // Two validators. // Spec starts with step 2. - header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero())); + header.set_difficulty(calculate_score(0, 1, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); @@ -1650,10 +1685,10 @@ mod tests { // Two validators. // Spec starts with step 2. header.set_seal(vec![encode(&5usize), encode(&(&*signature as &[u8]))]); - header.set_difficulty(calculate_score(U256::from(4), U256::from(5), U256::zero())); + header.set_difficulty(calculate_score(4, 5, 0)); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); header.set_seal(vec![encode(&3usize), encode(&(&*signature as &[u8]))]); - header.set_difficulty(calculate_score(U256::from(4), U256::from(3), U256::zero())); + header.set_difficulty(calculate_score(4, 3, 0)); assert!(engine.verify_block_family(&header, &parent_header).is_err()); } @@ -1687,7 +1722,7 @@ mod tests { parent_header.set_seal(vec![encode(&1usize)]); parent_header.set_gas_limit("222222".parse::().unwrap()); let mut header: Header = Header::default(); - header.set_difficulty(calculate_score(U256::from(1), U256::from(3), U256::zero())); + header.set_difficulty(calculate_score(1, 3, 0)); header.set_gas_limit("222222".parse::().unwrap()); header.set_seal(vec![encode(&3usize)]); @@ -1801,14 +1836,14 @@ mod tests { (spec, tap, accounts) } - fn empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> EmptyStep { + fn empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> EmptyStep { let empty_step_rlp = super::empty_step_rlp(step, parent_hash); let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into(); let parent_hash = parent_hash.clone(); EmptyStep { step, signature, parent_hash } } - fn sealed_empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> SealedEmptyStep { + fn sealed_empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> SealedEmptyStep { let empty_step_rlp = super::empty_step_rlp(step, parent_hash); let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into(); SealedEmptyStep { signature, step } @@ -1844,6 +1879,11 @@ mod tests { // we've received the message assert!(notify.messages.read().contains(&empty_step_rlp)); + let len = notify.messages.read().len(); + + // make sure that we don't generate empty step for the second time + assert_eq!(engine.generate_seal(b1.block(), &genesis_header), Seal::None); + assert_eq!(len, notify.messages.read().len()); } #[test] @@ -2058,7 +2098,7 @@ mod tests { let empty_step3 = sealed_empty_step(engine, 3, &parent_header.hash()); let empty_steps = vec![empty_step2, empty_step3]; - header.set_difficulty(calculate_score(U256::from(0), U256::from(4), U256::from(2))); + header.set_difficulty(calculate_score(0, 4, 2)); let signature = tap.sign(addr1, Some("1".into()), header.bare_hash()).unwrap(); header.set_seal(vec![ encode(&4usize), @@ -2173,4 +2213,52 @@ mod tests { BTreeMap::default(), ); } + + #[test] + fn test_empty_steps() { + let last_benign = Arc::new(AtomicUsize::new(0)); + let params = AuthorityRoundParams { + step_duration: 4, + start_step: Some(1), + validators: Box::new(TestSet::new(Default::default(), last_benign.clone())), + validate_score_transition: 0, + validate_step_transition: 0, + immediate_transitions: true, + maximum_uncle_count_transition: 0, + maximum_uncle_count: 0, + empty_steps_transition: 0, + maximum_empty_steps: 10, + block_reward: Default::default(), + block_reward_contract_transition: 0, + block_reward_contract: Default::default(), + }; + + let mut c_params = ::spec::CommonParams::default(); + c_params.gas_limit_bound_divisor = 5.into(); + let machine = ::machine::EthereumMachine::regular(c_params, Default::default()); + let engine = AuthorityRound::new(params, machine).unwrap(); + + + let parent_hash: H256 = 1.into(); + let signature = H520::default(); + let step = |step: u64| EmptyStep { + step, + parent_hash, + signature, + }; + + engine.handle_empty_step_message(step(1)); + engine.handle_empty_step_message(step(3)); + engine.handle_empty_step_message(step(2)); + engine.handle_empty_step_message(step(1)); + + assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(1), step(2), step(3)]); + assert_eq!(engine.empty_steps(2, 3, parent_hash), vec![]); + assert_eq!(engine.empty_steps(2, 4, parent_hash), vec![step(3)]); + + engine.clear_empty_steps(2); + + assert_eq!(engine.empty_steps(0, 3, parent_hash), vec![]); + assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(3)]); + } } diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index fefa9b5e0..d73a79777 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -139,6 +139,9 @@ extern crate trace_time; #[cfg_attr(test, macro_use)] extern crate evm; +#[cfg(test)] +extern crate env_logger; + pub extern crate ethstore; #[macro_use] diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 39dac1f2b..c73887800 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -576,7 +576,7 @@ impl Miner { trace!(target: "miner", "requires_reseal: sealing enabled"); // Disable sealing if there were no requests for SEALING_TIMEOUT_IN_BLOCKS - let had_requests = sealing.last_request.map(|last_request| + let had_requests = sealing.last_request.map(|last_request| best_block.saturating_sub(last_request) <= SEALING_TIMEOUT_IN_BLOCKS ).unwrap_or(false); diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 527b4e288..eb1a16278 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -65,6 +65,8 @@ pub enum Error { BadEpochProof(u64), /// Wrong chunk format. WrongChunkFormat(String), + /// Unlinked ancient block chain + UnlinkedAncientBlockChain, } impl fmt::Display for Error { @@ -91,6 +93,7 @@ impl fmt::Display for Error { Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."), Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i), Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg), + Error::UnlinkedAncientBlockChain => write!(f, "Unlinked ancient blocks chain"), } } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index fbe6d6a16..bd7e3cb51 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -56,7 +56,7 @@ use rand::{Rng, OsRng}; pub use self::error::Error; pub use self::consensus::*; -pub use self::service::{Service, DatabaseRestore}; +pub use self::service::{SnapshotClient, Service, DatabaseRestore}; pub use self::traits::SnapshotService; pub use self::watcher::Watcher; pub use types::snapshot_manifest::ManifestData; diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index 97fc516b8..f547faab1 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -22,12 +22,13 @@ use std::fs::{self, File}; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::cmp; use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE}; use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}; use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; -use client::{Client, ChainInfo, ClientIoMessage}; +use client::{BlockInfo, BlockChainClient, Client, ChainInfo, ClientIoMessage}; use engines::EthEngine; use error::{Error, ErrorKind as SnapshotErrorKind}; use snapshot::{Error as SnapshotError}; @@ -40,6 +41,7 @@ use ethereum_types::H256; use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use bytes::Bytes; use journaldb::Algorithm; +use kvdb::DBTransaction; use snappy; /// Helper for removing directories in case of error. @@ -203,6 +205,9 @@ impl Restoration { /// Type alias for client io channel. pub type Channel = IoChannel; +/// Trait alias for the Client Service used +pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore {} + /// Snapshot service parameters. pub struct ServiceParams { /// The consensus engine this is built on. @@ -219,7 +224,7 @@ pub struct ServiceParams { /// Usually "/snapshot" pub snapshot_root: PathBuf, /// A handle for database restoration. - pub db_restore: Arc, + pub client: Arc, } /// `SnapshotService` implementation. @@ -236,7 +241,7 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, - db_restore: Arc, + client: Arc, progress: super::Progress, taking_snapshot: AtomicBool, restoring_snapshot: AtomicBool, @@ -257,7 +262,7 @@ impl Service { genesis_block: params.genesis_block, state_chunks: AtomicUsize::new(0), block_chunks: AtomicUsize::new(0), - db_restore: params.db_restore, + client: params.client, progress: Default::default(), taking_snapshot: AtomicBool::new(false), restoring_snapshot: AtomicBool::new(false), @@ -334,12 +339,110 @@ impl Service { // replace one the client's database with our own. fn replace_client_db(&self) -> Result<(), Error> { - let our_db = self.restoration_db(); + let migrated_blocks = self.migrate_blocks()?; + trace!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks); - self.db_restore.restore_db(&*our_db.to_string_lossy())?; + let rest_db = self.restoration_db(); + self.client.restore_db(&*rest_db.to_string_lossy())?; Ok(()) } + // Migrate the blocks in the current DB into the new chain + fn migrate_blocks(&self) -> Result { + // Count the number of migrated blocks + let mut count = 0; + let rest_db = self.restoration_db(); + + let cur_chain_info = self.client.chain_info(); + + let next_db = self.restoration_db_handler.open(&rest_db)?; + let next_chain = BlockChain::new(Default::default(), &[], next_db.clone()); + let next_chain_info = next_chain.chain_info(); + + // The old database looks like this: + // [genesis, best_ancient_block] ... [first_block, best_block] + // If we are fully synced neither `best_ancient_block` nor `first_block` is set, and we can assume that the whole range from [genesis, best_block] is imported. + // The new database only contains the tip of the chain ([first_block, best_block]), + // so the useful set of blocks is defined as: + // [0 ... min(new.first_block, best_ancient_block or best_block)] + let find_range = || -> Option<(H256, H256)> { + let next_available_from = next_chain_info.first_block_number?; + let cur_available_to = cur_chain_info.ancient_block_number.unwrap_or(cur_chain_info.best_block_number); + + let highest_block_num = cmp::min(next_available_from.saturating_sub(1), cur_available_to); + + if highest_block_num == 0 { + return None; + } + + trace!(target: "snapshot", "Trying to import ancient blocks until {}", highest_block_num); + + // Here we start from the highest block number and go backward to 0, + // thus starting at `highest_block_num` and targetting `0`. + let target_hash = self.client.block_hash(BlockId::Number(0))?; + let start_hash = self.client.block_hash(BlockId::Number(highest_block_num))?; + + Some((start_hash, target_hash)) + }; + + let (start_hash, target_hash) = match find_range() { + Some(x) => x, + None => return Ok(0), + }; + + let mut batch = DBTransaction::new(); + let mut parent_hash = start_hash; + while parent_hash != target_hash { + // Early return if restoration is aborted + if !self.restoring_snapshot.load(Ordering::SeqCst) { + return Ok(count); + } + + let block = self.client.block(BlockId::Hash(parent_hash)).ok_or(::snapshot::error::Error::UnlinkedAncientBlockChain)?; + parent_hash = block.parent_hash(); + + let block_number = block.number(); + let block_receipts = self.client.block_receipts(&block.hash()); + let parent_total_difficulty = self.client.block_total_difficulty(BlockId::Hash(parent_hash)); + + match (block_receipts, parent_total_difficulty) { + (Some(block_receipts), Some(parent_total_difficulty)) => { + let block_receipts = block_receipts.receipts; + + next_chain.insert_unordered_block(&mut batch, block, block_receipts, Some(parent_total_difficulty), false, true); + count += 1; + }, + _ => break, + } + + // Writting changes to DB and logging every now and then + if block_number % 1_000 == 0 { + next_db.key_value().write_buffered(batch); + next_chain.commit(); + next_db.key_value().flush().expect("DB flush failed."); + batch = DBTransaction::new(); + } + + if block_number % 10_000 == 0 { + trace!(target: "snapshot", "Block restoration at #{}", block_number); + } + } + + // Final commit to the DB + next_db.key_value().write_buffered(batch); + next_chain.commit(); + next_db.key_value().flush().expect("DB flush failed."); + + // We couldn't reach the targeted hash + if parent_hash != target_hash { + return Err(::snapshot::error::Error::UnlinkedAncientBlockChain.into()); + } + + // Update best ancient block in the Next Chain + next_chain.update_best_ancient_block(&start_hash); + Ok(count) + } + /// Get a reference to the snapshot reader. pub fn reader(&self) -> RwLockReadGuard> { self.reader.read() @@ -480,12 +583,16 @@ impl Service { // Import previous chunks, continue if it fails self.import_prev_chunks(&mut res, manifest).ok(); - *self.status.lock() = RestorationStatus::Ongoing { - state_chunks: state_chunks as u32, - block_chunks: block_chunks as u32, - state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32, - block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32, - }; + // It could be that the restoration failed or completed in the meanwhile + let mut restoration_status = self.status.lock(); + if let RestorationStatus::Initializing { .. } = *restoration_status { + *restoration_status = RestorationStatus::Ongoing { + state_chunks: state_chunks as u32, + block_chunks: block_chunks as u32, + state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32, + block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32, + }; + } Ok(()) } @@ -752,26 +859,19 @@ impl Drop for Service { #[cfg(test)] mod tests { - use std::sync::Arc; use client::ClientIoMessage; use io::{IoService}; use spec::Spec; use journaldb::Algorithm; - use error::Error; use snapshot::{ManifestData, RestorationStatus, SnapshotService}; use super::*; use tempdir::TempDir; - use test_helpers::restoration_db_handler; - - struct NoopDBRestore; - impl DatabaseRestore for NoopDBRestore { - fn restore_db(&self, _new_db: &str) -> Result<(), Error> { - Ok(()) - } - } + use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; #[test] fn sends_async_messages() { + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices); let service = IoService::::start().unwrap(); let spec = Spec::new_test(); @@ -785,7 +885,7 @@ mod tests { pruning: Algorithm::Archive, channel: service.channel(), snapshot_root: dir, - db_restore: Arc::new(NoopDBRestore), + client: client, }; let service = Service::new(snapshot_params).unwrap(); diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/src/snapshot/tests/service.rs index a5af63b01..d98543cbb 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/src/snapshot/tests/service.rs @@ -16,26 +16,23 @@ //! Tests for the snapshot service. +use std::fs; use std::sync::Arc; use tempdir::TempDir; -use client::{Client, BlockInfo}; +use blockchain::BlockProvider; +use client::{Client, ClientConfig, ImportBlock, BlockInfo}; use ids::BlockId; +use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; use snapshot::service::{Service, ServiceParams}; -use snapshot::{self, ManifestData, SnapshotService}; +use snapshot::{chunk_state, chunk_secondary, ManifestData, Progress, SnapshotService, RestorationStatus}; use spec::Spec; -use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use parking_lot::Mutex; use io::IoChannel; use kvdb_rocksdb::DatabaseConfig; - -struct NoopDBRestore; - -impl snapshot::DatabaseRestore for NoopDBRestore { - fn restore_db(&self, _new_db: &str) -> Result<(), ::error::Error> { - Ok(()) - } -} +use verification::queue::kind::blocks::Unverified; #[test] fn restored_is_equivalent() { @@ -46,7 +43,6 @@ fn restored_is_equivalent() { const TX_PER: usize = 5; let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; - let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, TX_PER, &gas_prices); let tempdir = TempDir::new("").unwrap(); @@ -73,7 +69,7 @@ fn restored_is_equivalent() { pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: path, - db_restore: client2.clone(), + client: client2.clone(), }; let service = Service::new(service_params).unwrap(); @@ -94,7 +90,7 @@ fn restored_is_equivalent() { service.feed_block_chunk(hash, &chunk); } - assert_eq!(service.status(), ::snapshot::RestorationStatus::Inactive); + assert_eq!(service.status(), RestorationStatus::Inactive); for x in 0..NUM_BLOCKS { let block1 = client.block(BlockId::Number(x as u64)).unwrap(); @@ -106,6 +102,9 @@ fn restored_is_equivalent() { #[test] fn guards_delete_folders() { + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices); + let spec = Spec::new_null(); let tempdir = TempDir::new("").unwrap(); let service_params = ServiceParams { @@ -115,7 +114,7 @@ fn guards_delete_folders() { pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: tempdir.path().to_owned(), - db_restore: Arc::new(NoopDBRestore), + client: client, }; let service = Service::new(service_params).unwrap(); @@ -146,3 +145,201 @@ fn guards_delete_folders() { assert!(!path.join("db").exists()); assert!(path.join("temp").exists()); } + +#[test] +fn keep_ancient_blocks() { + ::env_logger::init().ok(); + + // Test variables + const NUM_BLOCKS: u64 = 500; + const NUM_SNAPSHOT_BLOCKS: u64 = 300; + const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; + + // Temporary folders + let tempdir = TempDir::new("").unwrap(); + let snapshot_path = tempdir.path().join("SNAP"); + + // Generate blocks + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let spec_f = Spec::new_null; + let spec = spec_f(); + let client = generate_dummy_client_with_spec_and_data(spec_f, NUM_BLOCKS as u32, 5, &gas_prices); + + let bc = client.chain(); + + // Create the Snapshot + let best_hash = bc.best_block_hash(); + let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); + let block_hashes = chunk_secondary( + Box::new(SNAPSHOT_MODE), + &bc, + best_hash, + &writer, + &Progress::default() + ).unwrap(); + let state_db = client.state_db().journal_db().boxed_clone(); + let start_header = bc.block_header_data(&best_hash).unwrap(); + let state_root = start_header.state_root(); + let state_hashes = chunk_state( + state_db.as_hashdb(), + &state_root, + &writer, + &Progress::default(), + None + ).unwrap(); + + let manifest = ::snapshot::ManifestData { + version: 2, + state_hashes: state_hashes, + state_root: state_root, + block_hashes: block_hashes, + block_number: NUM_BLOCKS, + block_hash: best_hash, + }; + + writer.into_inner().finish(manifest.clone()).unwrap(); + + // Initialize the Client + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let client_db = new_temp_db(&tempdir.path()); + let client2 = Client::new( + ClientConfig::default(), + &spec, + client_db, + Arc::new(::miner::Miner::new_for_tests(&spec, None)), + IoChannel::disconnected(), + ).unwrap(); + + // Add some ancient blocks + for block_number in 1..50 { + let block_hash = bc.block_hash(block_number).unwrap(); + let block = bc.block(&block_hash).unwrap(); + client2.import_block(Unverified::from_rlp(block.into_inner()).unwrap()).unwrap(); + } + + client2.import_verified_blocks(); + client2.flush_queue(); + + // Restore the Snapshot + let reader = PackedReader::new(&snapshot_path).unwrap().unwrap(); + let service_params = ServiceParams { + engine: spec.engine.clone(), + genesis_block: spec.genesis_block(), + restoration_db_handler: restoration_db_handler(db_config), + pruning: ::journaldb::Algorithm::Archive, + channel: IoChannel::disconnected(), + snapshot_root: tempdir.path().to_owned(), + client: client2.clone(), + }; + let service = Service::new(service_params).unwrap(); + service.init_restore(manifest.clone(), false).unwrap(); + + for hash in &manifest.block_hashes { + let chunk = reader.chunk(*hash).unwrap(); + service.feed_block_chunk(*hash, &chunk); + } + + for hash in &manifest.state_hashes { + let chunk = reader.chunk(*hash).unwrap(); + service.feed_state_chunk(*hash, &chunk); + } + + match service.status() { + RestorationStatus::Inactive => (), + RestorationStatus::Failed => panic!("Snapshot Restoration has failed."), + RestorationStatus::Ongoing { .. } => panic!("Snapshot Restoration should be done."), + _ => panic!("Invalid Snapshot Service status."), + } + + // Check that the latest block number is the right one + assert_eq!(client2.block(BlockId::Latest).unwrap().number(), NUM_BLOCKS as u64); + + // Check that we have blocks in [NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1 ; NUM_BLOCKS] + // but none before + assert!(client2.block(BlockId::Number(NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1)).is_some()); + assert!(client2.block(BlockId::Number(100)).is_none()); + + // Check that the first 50 blocks have been migrated + for block_number in 1..49 { + assert!(client2.block(BlockId::Number(block_number)).is_some()); + } +} + +#[test] +fn recover_aborted_recovery() { + ::env_logger::init().ok(); + + const NUM_BLOCKS: u32 = 400; + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, 5, &gas_prices); + + let spec = Spec::new_null(); + let tempdir = TempDir::new("").unwrap(); + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let client_db = new_db(); + let client2 = Client::new( + Default::default(), + &spec, + client_db, + Arc::new(::miner::Miner::new_for_tests(&spec, None)), + IoChannel::disconnected(), + ).unwrap(); + let service_params = ServiceParams { + engine: spec.engine.clone(), + genesis_block: spec.genesis_block(), + restoration_db_handler: restoration_db_handler(db_config), + pruning: ::journaldb::Algorithm::Archive, + channel: IoChannel::disconnected(), + snapshot_root: tempdir.path().to_owned(), + client: client2.clone(), + }; + + let service = Service::new(service_params).unwrap(); + service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap(); + + let manifest = service.manifest().unwrap(); + service.init_restore(manifest.clone(), true).unwrap(); + + // Restore only the state chunks + for hash in &manifest.state_hashes { + let chunk = service.chunk(*hash).unwrap(); + service.feed_state_chunk(*hash, &chunk); + } + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32); + assert_eq!(block_chunks_done, 0); + }, + e => panic!("Snapshot restoration must be ongoing ; {:?}", e), + } + + // Abort the restore... + service.abort_restore(); + + // And try again! + service.init_restore(manifest.clone(), true).unwrap(); + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32); + assert_eq!(block_chunks_done, 0); + }, + e => panic!("Snapshot restoration must be ongoing ; {:?}", e), + } + + // Remove the snapshot directory, and restart the restoration + // It shouldn't have restored any previous blocks + fs::remove_dir_all(tempdir.path()).unwrap(); + + // And try again! + service.init_restore(manifest.clone(), true).unwrap(); + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(block_chunks_done, 0); + assert_eq!(state_chunks_done, 0); + }, + _ => panic!("Snapshot restoration must be ongoing"), + } +} diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index bf9ab0204..9f00f9662 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -41,7 +41,7 @@ use transaction::{Action, Transaction, SignedTransaction}; use views::BlockView; use blooms_db; use kvdb::KeyValueDB; -use kvdb_rocksdb; +use kvdb_rocksdb::{self, Database, DatabaseConfig}; use tempdir::TempDir; use verification::queue::kind::blocks::Unverified; use encoded; @@ -263,30 +263,30 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> Arc { client } +struct TestBlockChainDB { + _blooms_dir: TempDir, + _trace_blooms_dir: TempDir, + blooms: blooms_db::Database, + trace_blooms: blooms_db::Database, + key_value: Arc, +} + +impl BlockChainDB for TestBlockChainDB { + fn key_value(&self) -> &Arc { + &self.key_value + } + + fn blooms(&self) -> &blooms_db::Database { + &self.blooms + } + + fn trace_blooms(&self) -> &blooms_db::Database { + &self.trace_blooms + } +} + /// Creates new test instance of `BlockChainDB` pub fn new_db() -> Arc { - struct TestBlockChainDB { - _blooms_dir: TempDir, - _trace_blooms_dir: TempDir, - blooms: blooms_db::Database, - trace_blooms: blooms_db::Database, - key_value: Arc, - } - - impl BlockChainDB for TestBlockChainDB { - fn key_value(&self) -> &Arc { - &self.key_value - } - - fn blooms(&self) -> &blooms_db::Database { - &self.blooms - } - - fn trace_blooms(&self) -> &blooms_db::Database { - &self.trace_blooms - } - } - let blooms_dir = TempDir::new("").unwrap(); let trace_blooms_dir = TempDir::new("").unwrap(); @@ -301,6 +301,26 @@ pub fn new_db() -> Arc { Arc::new(db) } +/// Creates a new temporary `BlockChainDB` on FS +pub fn new_temp_db(tempdir: &Path) -> Arc { + let blooms_dir = TempDir::new("").unwrap(); + let trace_blooms_dir = TempDir::new("").unwrap(); + let key_value_dir = tempdir.join("key_value"); + + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let key_value_db = Database::open(&db_config, key_value_dir.to_str().unwrap()).unwrap(); + + let db = TestBlockChainDB { + blooms: blooms_db::Database::open(blooms_dir.path()).unwrap(), + trace_blooms: blooms_db::Database::open(trace_blooms_dir.path()).unwrap(), + _blooms_dir: blooms_dir, + _trace_blooms_dir: trace_blooms_dir, + key_value: Arc::new(key_value_db) + }; + + Arc::new(db) +} + /// Creates new instance of KeyValueDBHandler pub fn restoration_db_handler(config: kvdb_rocksdb::DatabaseConfig) -> Box { struct RestorationDBHandler { diff --git a/ethcore/src/verification/queue/mod.rs b/ethcore/src/verification/queue/mod.rs index 9b1597439..b9242f47b 100644 --- a/ethcore/src/verification/queue/mod.rs +++ b/ethcore/src/verification/queue/mod.rs @@ -583,6 +583,13 @@ impl VerificationQueue { result } + /// Returns true if there is nothing currently in the queue. + /// TODO [ToDr] Optimize to avoid locking + pub fn is_empty(&self) -> bool { + let v = &self.verification; + v.unverified.lock().is_empty() && v.verifying.lock().is_empty() && v.verified.lock().is_empty() + } + /// Get queue status. pub fn queue_info(&self) -> QueueInfo { use std::mem::size_of; diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 8063ebecb..7474d79b3 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc, atomic}; use std::collections::{HashMap, BTreeMap}; use std::io; use std::ops::Range; @@ -33,10 +33,10 @@ use ethcore::client::{BlockChainClient, ChainNotify, ChainRoute, ChainMessageTyp use ethcore::snapshot::SnapshotService; use ethcore::header::BlockNumber; use sync_io::NetSyncIo; -use chain::{ChainSync, SyncStatus as EthSyncStatus}; +use chain::{ChainSyncApi, SyncStatus as EthSyncStatus}; use std::net::{SocketAddr, AddrParseError}; use std::str::FromStr; -use parking_lot::RwLock; +use parking_lot::{RwLock, Mutex}; use chain::{ETH_PROTOCOL_VERSION_63, ETH_PROTOCOL_VERSION_62, PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; @@ -228,6 +228,37 @@ impl AttachedProtocol { } } +/// A prioritized tasks run in a specialised timer. +/// Every task should be completed within a hard deadline, +/// if it's not it's either cancelled or split into multiple tasks. +/// NOTE These tasks might not complete at all, so anything +/// that happens here should work even if the task is cancelled. +#[derive(Debug)] +pub enum PriorityTask { + /// Propagate given block + PropagateBlock { + /// When the task was initiated + started: ::std::time::Instant, + /// Raw block RLP to propagate + block: Bytes, + /// Block hash + hash: H256, + /// Blocks difficulty + difficulty: U256, + }, + /// Propagate a list of transactions + PropagateTransactions(::std::time::Instant, Arc), +} +impl PriorityTask { + /// Mark the task as being processed, right after it's retrieved from the queue. + pub fn starting(&self) { + match *self { + PriorityTask::PropagateTransactions(_, ref is_ready) => is_ready.store(true, atomic::Ordering::SeqCst), + _ => {}, + } + } +} + /// EthSync initialization parameters. pub struct Params { /// Configuration. @@ -260,16 +291,16 @@ pub struct EthSync { subprotocol_name: [u8; 3], /// Light subprotocol name. light_subprotocol_name: [u8; 3], + /// Priority tasks notification channel + priority_tasks: Mutex>, } fn light_params( network_id: u64, - max_peers: u32, + median_peers: f64, pruning_info: PruningInfo, sample_store: Option>, ) -> LightParams { - const MAX_LIGHTSERV_LOAD: f64 = 0.5; - let mut light_params = LightParams { network_id: network_id, config: Default::default(), @@ -282,9 +313,7 @@ fn light_params( sample_store: sample_store, }; - let max_peers = ::std::cmp::max(max_peers, 1); - light_params.config.load_share = MAX_LIGHTSERV_LOAD / max_peers as f64; - + light_params.config.median_peers = median_peers; light_params } @@ -301,9 +330,10 @@ impl EthSync { .map(|mut p| { p.push("request_timings"); light_net::FileStore(p) }) .map(|store| Box::new(store) as Box<_>); + let median_peers = (params.network_config.min_peers + params.network_config.max_peers) as f64 / 2.0; let light_params = light_params( params.config.network_id, - params.network_config.max_peers, + median_peers, pruning_info, sample_store, ); @@ -315,13 +345,19 @@ impl EthSync { }) }; - let chain_sync = ChainSync::new(params.config, &*params.chain, params.private_tx_handler.clone()); + let (priority_tasks_tx, priority_tasks_rx) = mpsc::channel(); + let sync = ChainSyncApi::new( + params.config, + &*params.chain, + params.private_tx_handler.clone(), + priority_tasks_rx, + ); let service = NetworkService::new(params.network_config.clone().into_basic()?, connection_filter)?; let sync = Arc::new(EthSync { network: service, eth_handler: Arc::new(SyncProtocolHandler { - sync: RwLock::new(chain_sync), + sync, chain: params.chain, snapshot_service: params.snapshot_service, overlay: RwLock::new(HashMap::new()), @@ -330,26 +366,32 @@ impl EthSync { subprotocol_name: params.config.subprotocol_name, light_subprotocol_name: params.config.light_subprotocol_name, attached_protos: params.attached_protos, + priority_tasks: Mutex::new(priority_tasks_tx), }); Ok(sync) } + + /// Priority tasks producer + pub fn priority_tasks(&self) -> mpsc::Sender { + self.priority_tasks.lock().clone() + } } impl SyncProvider for EthSync { /// Get sync status fn status(&self) -> EthSyncStatus { - self.eth_handler.sync.read().status() + self.eth_handler.sync.status() } /// Get sync peers fn peers(&self) -> Vec { self.network.with_context_eval(self.subprotocol_name, |ctx| { let peer_ids = self.network.connected_peers(); - let eth_sync = self.eth_handler.sync.read(); let light_proto = self.light_proto.as_ref(); - peer_ids.into_iter().filter_map(|peer_id| { + let peer_info = self.eth_handler.sync.peer_info(&peer_ids); + peer_ids.into_iter().zip(peer_info).filter_map(|(peer_id, peer_info)| { let session_info = match ctx.session_info(peer_id) { None => return None, Some(info) => info, @@ -361,7 +403,7 @@ impl SyncProvider for EthSync { capabilities: session_info.peer_capabilities.into_iter().map(|c| c.to_string()).collect(), remote_address: session_info.remote_address, local_address: session_info.local_address, - eth_info: eth_sync.peer_info(&peer_id), + eth_info: peer_info, pip_info: light_proto.as_ref().and_then(|lp| lp.peer_status(peer_id)).map(Into::into), }) }).collect() @@ -373,17 +415,17 @@ impl SyncProvider for EthSync { } fn transactions_stats(&self) -> BTreeMap { - let sync = self.eth_handler.sync.read(); - sync.transactions_stats() - .iter() - .map(|(hash, stats)| (*hash, stats.into())) - .collect() + self.eth_handler.sync.transactions_stats() } } const PEERS_TIMER: TimerToken = 0; -const SYNC_TIMER: TimerToken = 1; -const TX_TIMER: TimerToken = 2; +const MAINTAIN_SYNC_TIMER: TimerToken = 1; +const CONTINUE_SYNC_TIMER: TimerToken = 2; +const TX_TIMER: TimerToken = 3; +const PRIORITY_TIMER: TimerToken = 4; + +pub(crate) const PRIORITY_TIMER_INTERVAL: Duration = Duration::from_millis(250); struct SyncProtocolHandler { /// Shared blockchain client. @@ -391,7 +433,7 @@ struct SyncProtocolHandler { /// Shared snapshot service. snapshot_service: Arc, /// Sync strategy - sync: RwLock, + sync: ChainSyncApi, /// Chain overlay used to cache data such as fork block. overlay: RwLock>, } @@ -400,13 +442,16 @@ impl NetworkProtocolHandler for SyncProtocolHandler { fn initialize(&self, io: &NetworkContext) { if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID { io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer"); - io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(MAINTAIN_SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(CONTINUE_SYNC_TIMER, Duration::from_millis(2500)).expect("Error registering sync timer"); io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer"); + + io.register_timer(PRIORITY_TIMER, PRIORITY_TIMER_INTERVAL).expect("Error registering peers timer"); } } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { - ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); + self.sync.dispatch_packet(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); } fn connected(&self, io: &NetworkContext, peer: &PeerId) { @@ -431,16 +476,28 @@ impl NetworkProtocolHandler for SyncProtocolHandler { let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay); match timer { PEERS_TIMER => self.sync.write().maintain_peers(&mut io), - SYNC_TIMER => self.sync.write().maintain_sync(&mut io), - TX_TIMER => { - self.sync.write().propagate_new_transactions(&mut io); - }, + MAINTAIN_SYNC_TIMER => self.sync.write().maintain_sync(&mut io), + CONTINUE_SYNC_TIMER => self.sync.write().continue_sync(&mut io), + TX_TIMER => self.sync.write().propagate_new_transactions(&mut io), + PRIORITY_TIMER => self.sync.process_priority_queue(&mut io), _ => warn!("Unknown timer {} triggered.", timer), } } } impl ChainNotify for EthSync { + fn block_pre_import(&self, bytes: &Bytes, hash: &H256, difficulty: &U256) { + let task = PriorityTask::PropagateBlock { + started: ::std::time::Instant::now(), + block: bytes.clone(), + hash: *hash, + difficulty: *difficulty, + }; + if let Err(e) = self.priority_tasks.lock().send(task) { + warn!(target: "sync", "Unexpected error during priority block propagation: {:?}", e); + } + } + fn new_blocks(&self, imported: Vec, invalid: Vec, @@ -940,19 +997,3 @@ impl LightSyncProvider for LightSync { Default::default() // TODO } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn light_params_load_share_depends_on_max_peers() { - let pruning_info = PruningInfo { - earliest_chain: 0, - earliest_state: 0, - }; - let params1 = light_params(0, 10, pruning_info.clone(), None); - let params2 = light_params(0, 20, pruning_info, None); - assert!(params1.config.load_share > params2.config.load_share) - } -} diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index e67029743..e1518d4f5 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -29,7 +29,6 @@ use rlp::Rlp; use snapshot::ChunkType; use std::cmp; use std::mem; -use std::collections::HashSet; use std::time::Instant; use sync_io::SyncIo; @@ -58,7 +57,6 @@ use super::{ SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, STATUS_PACKET, - TRANSACTIONS_PACKET, }; /// The Chain Sync Handler: handles responses from peers @@ -67,14 +65,9 @@ pub struct SyncHandler; impl SyncHandler { /// Handle incoming packet from peer pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) { - debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); - return; - } let rlp = Rlp::new(data); let result = match packet_id { STATUS_PACKET => SyncHandler::on_peer_status(sync, io, peer, &rlp), - TRANSACTIONS_PACKET => SyncHandler::on_peer_transactions(sync, io, peer, &rlp), BLOCK_HEADERS_PACKET => SyncHandler::on_peer_block_headers(sync, io, peer, &rlp), BLOCK_BODIES_PACKET => SyncHandler::on_peer_block_bodies(sync, io, peer, &rlp), RECEIPTS_PACKET => SyncHandler::on_peer_block_receipts(sync, io, peer, &rlp), @@ -104,15 +97,12 @@ impl SyncHandler { sync.sync_peer(io, peer, false); }, } - // give tasks to other peers - sync.continue_sync(io); } /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) { trace!(target: "sync", "Received consensus packet from {:?}", peer_id); io.chain().queue_consensus_message(r.as_raw().to_vec()); - Ok(()) } /// Called by peer when it is disconnecting @@ -578,8 +568,8 @@ impl SyncHandler { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: if sync.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, asking_snapshot_data: None, @@ -635,7 +625,7 @@ impl SyncHandler { } /// Called when peer sends us new transactions - fn on_peer_transactions(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), DownloaderImportError> { + pub fn on_peer_transactions(sync: &ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { // Accept transactions only when fully synced if !io.is_chain_queue_empty() || (sync.state != SyncState::Idle && sync.state != SyncState::NewBlocks) { trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id); diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 1a0f99b92..a01b25528 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -92,17 +92,17 @@ mod propagator; mod requester; mod supplier; -use std::sync::Arc; -use std::collections::{HashSet, HashMap}; +use std::sync::{Arc, mpsc}; +use std::collections::{HashSet, HashMap, BTreeMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; -use fastmap::H256FastMap; -use parking_lot::RwLock; +use fastmap::{H256FastMap, H256FastSet}; +use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; use bytes::Bytes; -use rlp::{Rlp, RlpStream, DecoderError}; +use rlp::{RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; @@ -112,7 +112,7 @@ use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, DownloadAction}; use rand::Rng; use snapshot::{Snapshot}; -use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID}; +use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID, PriorityTask}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; @@ -120,7 +120,7 @@ use transaction::UnverifiedTransaction; use self::handler::SyncHandler; use self::propagator::SyncPropagator; use self::requester::SyncRequester; -use self::supplier::SyncSupplier; +pub(crate) use self::supplier::SyncSupplier; known_heap_size!(0, PeerInfo); @@ -187,6 +187,11 @@ const FORK_HEADER_TIMEOUT: Duration = Duration::from_secs(3); const SNAPSHOT_MANIFEST_TIMEOUT: Duration = Duration::from_secs(5); const SNAPSHOT_DATA_TIMEOUT: Duration = Duration::from_secs(120); +/// Defines how much time we have to complete priority transaction or block propagation. +/// after the deadline is reached the task is considered finished +/// (so we might sent only to some part of the peers we originally intended to send to) +const PRIORITY_TASK_DEADLINE: Duration = Duration::from_millis(100); + #[derive(Copy, Clone, Eq, PartialEq, Debug)] /// Sync state pub enum SyncState { @@ -323,9 +328,9 @@ pub struct PeerInfo { /// Request timestamp ask_time: Instant, /// Holds a set of transactions recently sent to this peer to avoid spamming. - last_sent_transactions: HashSet, + last_sent_transactions: H256FastSet, /// Holds a set of private transactions and their signatures recently sent to this peer to avoid spamming. - last_sent_private_transactions: HashSet, + last_sent_private_transactions: H256FastSet, /// Pending request is expired and result should be ignored expired: bool, /// Peer fork confirmation status @@ -375,6 +380,217 @@ pub mod random { pub type RlpResponseResult = Result, PacketDecodeError>; pub type Peers = HashMap; +/// Thread-safe wrapper for `ChainSync`. +/// +/// NOTE always lock in order of fields declaration +pub struct ChainSyncApi { + /// Priority tasks queue + priority_tasks: Mutex>, + /// The rest of sync data + sync: RwLock, +} + +impl ChainSyncApi { + /// Creates new `ChainSyncApi` + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + priority_tasks: mpsc::Receiver, + ) -> Self { + ChainSyncApi { + sync: RwLock::new(ChainSync::new(config, chain, private_tx_handler)), + priority_tasks: Mutex::new(priority_tasks), + } + } + + /// Gives `write` access to underlying `ChainSync` + pub fn write(&self) -> RwLockWriteGuard { + self.sync.write() + } + + /// Returns info about given list of peers + pub fn peer_info(&self, ids: &[PeerId]) -> Vec> { + let sync = self.sync.read(); + ids.iter().map(|id| sync.peer_info(id)).collect() + } + + /// Returns synchonization status + pub fn status(&self) -> SyncStatus { + self.sync.read().status() + } + + /// Returns transactions propagation statistics + pub fn transactions_stats(&self) -> BTreeMap { + self.sync.read().transactions_stats() + .iter() + .map(|(hash, stats)| (*hash, stats.into())) + .collect() + } + + /// Dispatch incoming requests and responses + pub fn dispatch_packet(&self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + SyncSupplier::dispatch_packet(&self.sync, io, peer, packet_id, data) + } + + /// Process a priority propagation queue. + /// This task is run from a timer and should be time constrained. + /// Hence we set up a deadline for the execution and cancel the task if the deadline is exceeded. + /// + /// NOTE This method should only handle stuff that can be canceled and would reach other peers + /// by other means. + pub fn process_priority_queue(&self, io: &mut SyncIo) { + fn check_deadline(deadline: Instant) -> Option { + let now = Instant::now(); + if now > deadline { + None + } else { + Some(deadline - now) + } + } + + // deadline to get the task from the queue + let deadline = Instant::now() + ::api::PRIORITY_TIMER_INTERVAL; + let mut work = || { + let task = { + let tasks = self.priority_tasks.try_lock_until(deadline)?; + let left = check_deadline(deadline)?; + tasks.recv_timeout(left).ok()? + }; + task.starting(); + // wait for the sync lock until deadline, + // note we might drop the task here if we won't manage to acquire the lock. + let mut sync = self.sync.try_write_until(deadline)?; + // since we already have everything let's use a different deadline + // to do the rest of the job now, so that previous work is not wasted. + let deadline = Instant::now() + PRIORITY_TASK_DEADLINE; + let as_ms = move |prev| { + let dur: Duration = Instant::now() - prev; + dur.as_secs() * 1_000 + dur.subsec_millis() as u64 + }; + match task { + // NOTE We can't simply use existing methods, + // cause the block is not in the DB yet. + PriorityTask::PropagateBlock { started, block, hash, difficulty } => { + // try to send to peers that are on the same block as us + // (they will most likely accept the new block). + let chain_info = io.chain().chain_info(); + let total_difficulty = chain_info.total_difficulty + difficulty; + let rlp = ChainSync::create_block_rlp(&block, total_difficulty); + for peers in sync.get_peers(&chain_info, PeerState::SameBlock).chunks(10) { + check_deadline(deadline)?; + for peer in peers { + SyncPropagator::send_packet(io, *peer, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer) { + peer.latest_hash = hash; + } + } + } + debug!(target: "sync", "Finished block propagation, took {}ms", as_ms(started)); + }, + PriorityTask::PropagateTransactions(time, _) => { + SyncPropagator::propagate_new_transactions(&mut sync, io, || { + check_deadline(deadline).is_some() + }); + debug!(target: "sync", "Finished transaction propagation, took {}ms", as_ms(time)); + }, + } + + Some(()) + }; + + // Process as many items as we can until the deadline is reached. + loop { + if work().is_none() { + return; + } + } + } +} + +// Static methods +impl ChainSync { + /// creates rlp to send for the tree defined by 'from' and 'to' hashes + fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { + match chain.tree_route(from, to) { + Some(route) => { + let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); + match route.blocks.len() { + 0 => None, + _ => { + let mut blocks = route.blocks; + blocks.extend(uncles); + let mut rlp_stream = RlpStream::new_list(blocks.len()); + for block_hash in blocks { + let mut hash_rlp = RlpStream::new_list(2); + let number = chain.block_header(BlockId::Hash(block_hash.clone())) + .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); + hash_rlp.append(&block_hash); + hash_rlp.append(&number); + rlp_stream.append_raw(hash_rlp.as_raw(), 1); + } + Some(rlp_stream.out()) + } + } + }, + None => None + } + } + + /// creates rlp from block bytes and total difficulty + fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { + let mut rlp_stream = RlpStream::new_list(2); + rlp_stream.append_raw(bytes, 1); + rlp_stream.append(&total_difficulty); + rlp_stream.out() + } + + /// creates latest block rlp for the given client + fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) + .expect("Best block always exists").into_inner(), + chain.chain_info().total_difficulty + ) + } + + /// creates given hash block rlp for the given client + fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), + chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") + ) + } + + fn select_random_peers(peers: &[PeerId]) -> Vec { + // take sqrt(x) peers + let mut peers = peers.to_vec(); + let mut count = (peers.len() as f64).powf(0.5).round() as usize; + count = cmp::min(count, MAX_PEERS_PROPAGATION); + count = cmp::max(count, MIN_PEERS_PROPAGATION); + random::new().shuffle(&mut peers); + peers.truncate(count); + peers + } + + fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { + let best_block = chain.chain_info().best_block_number; + match warp_sync { + WarpSync::Enabled => SyncState::WaitingPeers, + WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, + _ => SyncState::Idle, + } + } +} + +/// A peer query method for getting a list of peers +enum PeerState { + /// Peer is on different hash than us + Lagging, + /// Peer is on the same block as us + SameBlock +} + /// Blockchain sync handler. /// See module documentation for more details. pub struct ChainSync { @@ -417,10 +633,14 @@ pub struct ChainSync { impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new(config: SyncConfig, chain: &BlockChainClient, private_tx_handler: Arc) -> ChainSync { + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + ) -> Self { let chain_info = chain.chain_info(); let best_block = chain.chain_info().best_block_number; - let state = ChainSync::get_init_state(config.warp_sync, chain); + let state = Self::get_init_state(config.warp_sync, chain); let mut sync = ChainSync { state, @@ -445,15 +665,6 @@ impl ChainSync { sync } - fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { - let best_block = chain.chain_info().best_block_number; - match warp_sync { - WarpSync::Enabled => SyncState::WaitingPeers, - WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, - _ => SyncState::Idle, - } - } - /// Returns synchonization status pub fn status(&self) -> SyncStatus { let last_imported_number = self.new_blocks.last_imported_block_number(); @@ -521,7 +732,7 @@ impl ChainSync { } } } - self.state = state.unwrap_or_else(|| ChainSync::get_init_state(self.warp_sync, io.chain())); + self.state = state.unwrap_or_else(|| Self::get_init_state(self.warp_sync, io.chain())); // Reactivate peers only if some progress has been made // since the last sync round of if starting fresh. self.active_peers = self.peers.keys().cloned().collect(); @@ -655,37 +866,35 @@ impl ChainSync { } /// Resume downloading - fn continue_sync(&mut self, io: &mut SyncIo) { - // Collect active peers that can sync - let confirmed_peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| - if peer.can_sync() { - Some((*peer_id, peer.protocol_version)) - } else { - None - } - ).collect(); - - trace!( - target: "sync", - "Syncing with peers: {} active, {} confirmed, {} total", - self.active_peers.len(), confirmed_peers.len(), self.peers.len() - ); - + pub fn continue_sync(&mut self, io: &mut SyncIo) { if self.state == SyncState::Waiting { trace!(target: "sync", "Waiting for the block queue"); } else if self.state == SyncState::SnapshotWaiting { trace!(target: "sync", "Waiting for the snapshot restoration"); } else { - let mut peers: Vec<(PeerId, u8)> = confirmed_peers.iter().filter(|&&(peer_id, _)| - self.active_peers.contains(&peer_id) - ).map(|v| *v).collect(); + // Collect active peers that can sync + let mut peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| + if peer.can_sync() && peer.asking == PeerAsking::Nothing && self.active_peers.contains(&peer_id) { + Some((*peer_id, peer.protocol_version)) + } else { + None + } + ).collect(); - random::new().shuffle(&mut peers); //TODO: sort by rating - // prefer peers with higher protocol version - peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); + if peers.len() > 0 { + trace!( + target: "sync", + "Syncing with peers: {} active, {} available, {} total", + self.active_peers.len(), peers.len(), self.peers.len() + ); - for (peer_id, _) in peers { - self.sync_peer(io, peer_id, false); + random::new().shuffle(&mut peers); // TODO (#646): sort by rating + // prefer peers with higher protocol version + peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); + + for (peer_id, _) in peers { + self.sync_peer(io, peer_id, false); + } } } @@ -970,6 +1179,12 @@ impl ChainSync { self.state = SyncState::Blocks; self.continue_sync(io); }, + SyncState::SnapshotData => match io.snapshot_service().status() { + RestorationStatus::Inactive | RestorationStatus::Failed => { + self.state = SyncState::SnapshotWaiting; + }, + RestorationStatus::Initializing { .. } | RestorationStatus::Ongoing { .. } => (), + }, SyncState::SnapshotWaiting => { match io.snapshot_service().status() { RestorationStatus::Inactive => { @@ -998,67 +1213,24 @@ impl ChainSync { } } - /// creates rlp to send for the tree defined by 'from' and 'to' hashes - fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { - match chain.tree_route(from, to) { - Some(route) => { - let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); - match route.blocks.len() { - 0 => None, - _ => { - let mut blocks = route.blocks; - blocks.extend(uncles); - let mut rlp_stream = RlpStream::new_list(blocks.len()); - for block_hash in blocks { - let mut hash_rlp = RlpStream::new_list(2); - let number = chain.block_header(BlockId::Hash(block_hash.clone())) - .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); - hash_rlp.append(&block_hash); - hash_rlp.append(&number); - rlp_stream.append_raw(hash_rlp.as_raw(), 1); - } - Some(rlp_stream.out()) - } - } - }, - None => None - } + /// returns peer ids that have different block than our chain + fn get_lagging_peers(&self, chain_info: &BlockChainInfo) -> Vec { + self.get_peers(chain_info, PeerState::Lagging) } - /// creates rlp from block bytes and total difficulty - fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { - let mut rlp_stream = RlpStream::new_list(2); - rlp_stream.append_raw(bytes, 1); - rlp_stream.append(&total_difficulty); - rlp_stream.out() - } - - /// creates latest block rlp for the given client - fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) - .expect("Best block always exists").into_inner(), - chain.chain_info().total_difficulty - ) - } - - /// creates given hash block rlp for the given client - fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), - chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") - ) - } - - /// returns peer ids that have different blocks than our chain - fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo) -> Vec { + /// returns peer ids that have different or the same blocks than our chain + fn get_peers(&self, chain_info: &BlockChainInfo, peers: PeerState) -> Vec { let latest_hash = chain_info.best_block_hash; self .peers - .iter_mut() + .iter() .filter_map(|(&id, ref mut peer_info)| { trace!(target: "sync", "Checking peer our best {} their best {}", latest_hash, peer_info.latest_hash); - if peer_info.latest_hash != latest_hash { + let matches = match peers { + PeerState::Lagging => peer_info.latest_hash != latest_hash, + PeerState::SameBlock => peer_info.latest_hash == latest_hash, + }; + if matches { Some(id) } else { None @@ -1067,17 +1239,6 @@ impl ChainSync { .collect::>() } - fn select_random_peers(peers: &[PeerId]) -> Vec { - // take sqrt(x) peers - let mut peers = peers.to_vec(); - let mut count = (peers.len() as f64).powf(0.5).round() as usize; - count = cmp::min(count, MAX_PEERS_PROPAGATION); - count = cmp::max(count, MIN_PEERS_PROPAGATION); - random::new().shuffle(&mut peers); - peers.truncate(count); - peers - } - fn get_consensus_peers(&self) -> Vec { self.peers.iter().filter_map(|(id, p)| if p.protocol_version >= PAR_PROTOCOL_VERSION_2.0 { Some(*id) } else { None }).collect() } @@ -1126,21 +1287,10 @@ impl ChainSync { } } - /// Dispatch incoming requests and responses - pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) - } - pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); SyncHandler::on_packet(self, io, peer, packet_id, data); } - /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - SyncHandler::on_consensus_packet(io, peer_id, r) - } - /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { SyncHandler::on_peer_aborting(self, io, peer); @@ -1152,8 +1302,16 @@ impl ChainSync { } /// propagates new transactions to all peers - pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize { - SyncPropagator::propagate_new_transactions(self, io) + pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) { + let deadline = Instant::now() + Duration::from_millis(500); + SyncPropagator::propagate_new_transactions(self, io, || { + if deadline > Instant::now() { + true + } else { + debug!(target: "sync", "Wasn't able to finish transaction propagation within a deadline."); + false + } + }); } /// Broadcast consensus message to peers. @@ -1169,7 +1327,7 @@ impl ChainSync { #[cfg(test)] pub mod tests { - use std::collections::{HashSet, VecDeque}; + use std::collections::{VecDeque}; use ethkey; use network::PeerId; use tests::helpers::{TestIo}; @@ -1285,8 +1443,8 @@ pub mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: super::ForkConfirmation::Confirmed, snapshot_number: None, @@ -1301,7 +1459,7 @@ pub mod tests { fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); client.add_blocks(100, EachBlockWith::Uncle); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); + let sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); let chain_info = client.chain_info(); let lagging_peers = sync.get_lagging_peers(&chain_info); @@ -1441,3 +1599,4 @@ pub mod tests { assert_eq!(status.status.transaction_count, 0); } } + diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 82b592c4b..689ccfc02 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -18,6 +18,7 @@ use bytes::Bytes; use ethereum_types::H256; use ethcore::client::BlockChainInfo; use ethcore::header::BlockNumber; +use fastmap::H256FastSet; use network::{PeerId, PacketId}; use rand::Rng; use rlp::{Encodable, RlpStream}; @@ -69,49 +70,51 @@ impl SyncPropagator { /// propagates latest block to a set of peers pub fn propagate_blocks(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewBlocks to {:?}", peers); - let mut sent = 0; - for peer_id in peers { - if blocks.is_empty() { - let rlp = ChainSync::create_latest_block_rlp(io.chain()); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); - } else { - for h in blocks { - let rlp = ChainSync::create_new_block_rlp(io.chain(), h); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); + let sent = peers.len(); + let mut send_packet = |io: &mut SyncIo, rlp: Bytes| { + for peer_id in peers { + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = chain_info.best_block_hash.clone(); } } - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); + }; + + if blocks.is_empty() { + let rlp = ChainSync::create_latest_block_rlp(io.chain()); + send_packet(io, rlp); + } else { + for h in blocks { + let rlp = ChainSync::create_new_block_rlp(io.chain(), h); + send_packet(io, rlp); } - sent += 1; } + sent } /// propagates new known hashes to all peers pub fn propagate_new_hashes(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewHashes to {:?}", peers); - let mut sent = 0; let last_parent = *io.chain().best_block_header().parent_hash(); + let best_block_hash = chain_info.best_block_hash; + let rlp = match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &best_block_hash) { + Some(rlp) => rlp, + None => return 0 + }; + + let sent = peers.len(); for peer_id in peers { - sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) { - Some(rlp) => { - { - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); - } - } - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp); - 1 - }, - None => 0 + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = best_block_hash; } + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp.clone()); } sent } /// propagates new transactions to all peers - pub fn propagate_new_transactions(sync: &mut ChainSync, io: &mut SyncIo) -> usize { + pub fn propagate_new_transactions bool>(sync: &mut ChainSync, io: &mut SyncIo, mut should_continue: F) -> usize { // Early out if nobody to send to. if sync.peers.is_empty() { return 0; @@ -122,6 +125,10 @@ impl SyncPropagator { return 0; } + if !should_continue() { + return 0; + } + let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter() .map(|tx| tx.signed()) .partition(|tx| !tx.gas_price.is_zero()); @@ -130,24 +137,34 @@ impl SyncPropagator { let mut affected_peers = HashSet::new(); if !transactions.is_empty() { let peers = SyncPropagator::select_peers_for_transactions(sync, |_| true); - affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, peers, transactions); + affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, peers, transactions, &mut should_continue, + ); } // most of times service_transactions will be empty // => there's no need to merge packets if !service_transactions.is_empty() { let service_transactions_peers = SyncPropagator::select_peers_for_transactions(sync, |peer_id| accepts_service_transaction(&io.peer_info(*peer_id))); - let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, service_transactions_peers, service_transactions); + let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, service_transactions_peers, service_transactions, &mut should_continue + ); affected_peers.extend(&service_transactions_affected_peers); } affected_peers.len() } - fn propagate_transactions_to_peers(sync: &mut ChainSync, io: &mut SyncIo, peers: Vec, transactions: Vec<&SignedTransaction>) -> HashSet { + fn propagate_transactions_to_peers bool>( + sync: &mut ChainSync, + io: &mut SyncIo, + peers: Vec, + transactions: Vec<&SignedTransaction>, + mut should_continue: F, + ) -> HashSet { let all_transactions_hashes = transactions.iter() .map(|tx| tx.hash()) - .collect::>(); + .collect::(); let all_transactions_rlp = { let mut packet = RlpStream::new_list(transactions.len()); for tx in &transactions { packet.append(&**tx); } @@ -157,102 +174,104 @@ impl SyncPropagator { // Clear old transactions from stats sync.transactions_stats.retain(&all_transactions_hashes); - // sqrt(x)/x scaled to max u32 - let block_number = io.chain().chain_info().best_block_number; - - let lucky_peers = { - peers.into_iter() - .filter_map(|peer_id| { - let stats = &mut sync.transactions_stats; - let peer_info = sync.peers.get_mut(&peer_id) - .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - - // Send all transactions - if peer_info.last_sent_transactions.is_empty() { - // update stats - for hash in &all_transactions_hashes { - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - stats.propagated(hash, id, block_number); - } - peer_info.last_sent_transactions = all_transactions_hashes.clone(); - return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone())); - } - - // Get hashes of all transactions to send to this peer - let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) - .cloned() - .collect::>(); - if to_send.is_empty() { - return None; - } - - // Construct RLP - let (packet, to_send) = { - let mut to_send = to_send; - let mut packet = RlpStream::new(); - packet.begin_unbounded_list(); - let mut pushed = 0; - for tx in &transactions { - let hash = tx.hash(); - if to_send.contains(&hash) { - let mut transaction = RlpStream::new(); - tx.rlp_append(&mut transaction); - let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); - if !appended { - // Maximal packet size reached just proceed with sending - debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); - to_send = to_send.into_iter().take(pushed).collect(); - break; - } - pushed += 1; - } - } - packet.complete_unbounded_list(); - (packet, to_send) - }; - - // Update stats - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - for hash in &to_send { - // update stats - stats.propagated(hash, id, block_number); - } - - peer_info.last_sent_transactions = all_transactions_hashes - .intersection(&peer_info.last_sent_transactions) - .chain(&to_send) - .cloned() - .collect(); - Some((peer_id, to_send.len(), packet.out())) - }) - .collect::>() + let send_packet = |io: &mut SyncIo, peer_id: PeerId, sent: usize, rlp: Bytes| { + let size = rlp.len(); + SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); + trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); }; - // Send RLPs - let mut peers = HashSet::new(); - if lucky_peers.len() > 0 { - let mut max_sent = 0; - let lucky_peers_len = lucky_peers.len(); - for (peer_id, sent, rlp) in lucky_peers { - peers.insert(peer_id); - let size = rlp.len(); - SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); - trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); - max_sent = cmp::max(max_sent, sent); + let block_number = io.chain().chain_info().best_block_number; + let mut sent_to_peers = HashSet::new(); + let mut max_sent = 0; + + // for every peer construct and send transactions packet + for peer_id in peers { + if !should_continue() { + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + return sent_to_peers; } - debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len); + + let stats = &mut sync.transactions_stats; + let peer_info = sync.peers.get_mut(&peer_id) + .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); + + // Send all transactions, if the peer doesn't know about anything + if peer_info.last_sent_transactions.is_empty() { + // update stats + for hash in &all_transactions_hashes { + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + stats.propagated(hash, id, block_number); + } + peer_info.last_sent_transactions = all_transactions_hashes.clone(); + + send_packet(io, peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, all_transactions_hashes.len()); + continue; + } + + // Get hashes of all transactions to send to this peer + let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) + .cloned() + .collect::>(); + if to_send.is_empty() { + continue; + } + + // Construct RLP + let (packet, to_send) = { + let mut to_send = to_send; + let mut packet = RlpStream::new(); + packet.begin_unbounded_list(); + let mut pushed = 0; + for tx in &transactions { + let hash = tx.hash(); + if to_send.contains(&hash) { + let mut transaction = RlpStream::new(); + tx.rlp_append(&mut transaction); + let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); + if !appended { + // Maximal packet size reached just proceed with sending + debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); + to_send = to_send.into_iter().take(pushed).collect(); + break; + } + pushed += 1; + } + } + packet.complete_unbounded_list(); + (packet, to_send) + }; + + // Update stats + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + for hash in &to_send { + // update stats + stats.propagated(hash, id, block_number); + } + + peer_info.last_sent_transactions = all_transactions_hashes + .intersection(&peer_info.last_sent_transactions) + .chain(&to_send) + .cloned() + .collect(); + send_packet(io, peer_id, to_send.len(), packet.out()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, to_send.len()); + } - peers + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + sent_to_peers } pub fn propagate_latest_blocks(sync: &mut ChainSync, io: &mut SyncIo, sealed: &[H256]) { let chain_info = io.chain().chain_info(); if (((chain_info.best_block_number as i64) - (sync.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let mut peers = sync.get_lagging_peers(&chain_info); + let peers = sync.get_lagging_peers(&chain_info); if sealed.is_empty() { let hashes = SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers); - peers = ChainSync::select_random_peers(&peers); + let peers = ChainSync::select_random_peers(&peers); let blocks = SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); @@ -318,7 +337,7 @@ impl SyncPropagator { } /// Generic packet sender - fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { + pub fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { if let Err(e) = sync.send(peer_id, packet_id, packet) { debug!(target:"sync", "Error sending packet: {:?}", e); sync.disconnect_peer(peer_id); @@ -419,8 +438,8 @@ mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: ForkConfirmation::Confirmed, snapshot_number: None, @@ -447,13 +466,13 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Even after new block transactions should not be propagated twice sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the third time - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -474,7 +493,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); io.chain.insert_transaction_to_queue(); // New block import should not trigger propagation. // (we only propagate on timeout) @@ -498,10 +517,10 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(0, io.packets.len()); assert_eq!(0, peer_count); @@ -519,7 +538,7 @@ mod tests { // should sent some { let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(1, io.packets.len()); assert_eq!(1, peer_count); } @@ -528,9 +547,9 @@ mod tests { let (peer_count2, peer_count3) = { let mut io = TestIo::new(&mut client, &ss, &queue, None); // Propagate new transactions - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // And now the peer should have all transactions - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); (peer_count2, peer_count3) }; @@ -553,7 +572,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); let stats = sync.transactions_stats(); assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") @@ -583,7 +602,7 @@ mod tests { io.peers_info.insert(4, "Parity-Ethereum/v2.7.3-ABCDEFGH".to_owned()); // and new service transaction is propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // peer#2 && peer#4 are receiving service transaction assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET @@ -607,7 +626,7 @@ mod tests { io.peers_info.insert(1, "Parity-Ethereum/v2.6".to_owned()); // and service + non-service transactions are propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // two separate packets for peer are queued: // 1) with non-service-transaction diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 201e0d9f3..eaee584ca 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -27,6 +27,7 @@ use sync_io::SyncIo; use super::{ ChainSync, + SyncHandler, RlpResponseResult, PacketDecodeError, BLOCK_BODIES_PACKET, @@ -47,6 +48,8 @@ use super::{ RECEIPTS_PACKET, SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, + STATUS_PACKET, + TRANSACTIONS_PACKET, }; /// The Chain Sync Supplier: answers requests from peers with available data @@ -56,6 +59,7 @@ impl SyncSupplier { /// Dispatch incoming requests and responses pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { let rlp = Rlp::new(data); + let result = match packet_id { GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_block_bodies, @@ -80,9 +84,39 @@ impl SyncSupplier { GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_snapshot_data, |e| format!("Error sending snapshot data: {:?}", e)), - CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), - _ => { + + STATUS_PACKET => { sync.write().on_packet(io, peer, packet_id, data); + Ok(()) + }, + // Packets that require the peer to be confirmed + _ => { + if !sync.read().peers.contains_key(&peer) { + debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); + return; + } + debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); + + match packet_id { + CONSENSUS_DATA_PACKET => { + SyncHandler::on_consensus_packet(io, peer, &rlp) + }, + TRANSACTIONS_PACKET => { + let res = { + let sync_ro = sync.read(); + SyncHandler::on_peer_transactions(&*sync_ro, io, peer, &rlp) + }; + if res.is_err() { + // peer sent invalid data, disconnect. + io.disable_peer(peer); + sync.write().deactivate_peer(io, peer); + } + }, + _ => { + sync.write().on_packet(io, peer, packet_id, data); + } + } + Ok(()) } }; @@ -226,7 +260,8 @@ impl SyncSupplier { let mut added_receipts = 0usize; let mut data = Bytes::new(); for i in 0..count { - if let Some(mut receipts_bytes) = io.chain().encoded_block_receipts(&rlp.val_at::(i)?) { + if let Some(receipts) = io.chain().block_receipts(&rlp.val_at::(i)?) { + let mut receipts_bytes = ::rlp::encode(&receipts); data.append(&mut receipts_bytes); added_receipts += receipts_bytes.len(); added_headers += 1; @@ -403,7 +438,7 @@ mod test { io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); assert_eq!(1, io.packets.len()); } @@ -445,7 +480,7 @@ mod test { assert_eq!(603, rlp_result.unwrap().1.out().len()); io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); assert_eq!(1, io.packets.len()); } } diff --git a/ethcore/sync/src/light_sync/mod.rs b/ethcore/sync/src/light_sync/mod.rs index b4000c082..cda250f12 100644 --- a/ethcore/sync/src/light_sync/mod.rs +++ b/ethcore/sync/src/light_sync/mod.rs @@ -34,6 +34,7 @@ use std::collections::{HashMap, HashSet}; use std::mem; +use std::ops::Deref; use std::sync::Arc; use std::time::{Instant, Duration}; @@ -213,6 +214,44 @@ enum SyncState { Rounds(SyncRound), } +/// A wrapper around the SyncState that makes sure to +/// update the giving reference to `is_idle` +#[derive(Debug)] +struct SyncStateWrapper { + state: SyncState, +} + +impl SyncStateWrapper { + /// Create a new wrapper for SyncState::Idle + pub fn idle() -> Self { + SyncStateWrapper { + state: SyncState::Idle, + } + } + + /// Set the new state's value, making sure `is_idle` gets updated + pub fn set(&mut self, state: SyncState, is_idle_handle: &mut bool) { + *is_idle_handle = match state { + SyncState::Idle => true, + _ => false, + }; + self.state = state; + } + + /// Returns the internal state's value + pub fn into_inner(self) -> SyncState { + self.state + } +} + +impl Deref for SyncStateWrapper { + type Target = SyncState; + + fn deref(&self) -> &SyncState { + &self.state + } +} + struct ResponseCtx<'a> { peer: PeerId, req_id: ReqId, @@ -235,7 +274,9 @@ pub struct LightSync { pending_reqs: Mutex>, // requests from this handler client: Arc, rng: Mutex, - state: Mutex, + state: Mutex, + // We duplicate this state tracking to avoid deadlocks in `is_major_importing`. + is_idle: Mutex, } #[derive(Debug, Clone)] @@ -309,16 +350,17 @@ impl Handler for LightSync { if new_best.is_none() { debug!(target: "sync", "No peers remain. Reverting to idle"); - *self.state.lock() = SyncState::Idle; + self.set_state(&mut self.state.lock(), SyncState::Idle); } else { let mut state = self.state.lock(); - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.requests_abandoned(unfulfilled)), SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(unfulfilled)), }; + self.set_state(&mut state, next_state); } self.maintain_sync(ctx.as_basic()); @@ -390,12 +432,13 @@ impl Handler for LightSync { data: headers, }; - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.process_response(&ctx, &*self.client)), SyncState::Rounds(round) => SyncState::Rounds(round.process_response(&ctx)), }; + self.set_state(&mut state, next_state); } self.maintain_sync(ctx.as_basic()); @@ -408,12 +451,18 @@ impl Handler for LightSync { // private helpers impl LightSync { + /// Sets the LightSync's state, and update + /// `is_idle` + fn set_state(&self, state: &mut SyncStateWrapper, next_state: SyncState) { + state.set(next_state, &mut self.is_idle.lock()); + } + // Begins a search for the common ancestor and our best block. // does not lock state, instead has a mutable reference to it passed. - fn begin_search(&self, state: &mut SyncState) { + fn begin_search(&self, state: &mut SyncStateWrapper) { if let None = *self.best_seen.lock() { // no peers. - *state = SyncState::Idle; + self.set_state(state, SyncState::Idle); return; } @@ -422,7 +471,8 @@ impl LightSync { trace!(target: "sync", "Beginning search for common ancestor from {:?}", (chain_info.best_block_number, chain_info.best_block_hash)); - *state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number)); + let next_state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number)); + self.set_state(state, next_state); } // handles request dispatch, block import, state machine transitions, and timeouts. @@ -435,7 +485,7 @@ impl LightSync { let chain_info = client.chain_info(); let mut state = self.state.lock(); - debug!(target: "sync", "Maintaining sync ({:?})", &*state); + debug!(target: "sync", "Maintaining sync ({:?})", **state); // drain any pending blocks into the queue. { @@ -445,11 +495,12 @@ impl LightSync { loop { if client.queue_info().is_full() { break } - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(round) => SyncState::Rounds(round.drain(&mut sink, Some(DRAIN_AMOUNT))), other => other, }; + self.set_state(&mut state, next_state); if sink.is_empty() { break } trace!(target: "sync", "Drained {} headers to import", sink.len()); @@ -483,15 +534,15 @@ impl LightSync { let network_score = other.as_ref().map(|target| target.head_td); trace!(target: "sync", "No target to sync to. Network score: {:?}, Local score: {:?}", network_score, best_td); - *state = SyncState::Idle; + self.set_state(&mut state, SyncState::Idle); return; } }; - match mem::replace(&mut *state, SyncState::Idle) { + match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(SyncRound::Abort(reason, remaining)) => { if remaining.len() > 0 { - *state = SyncState::Rounds(SyncRound::Abort(reason, remaining)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::Abort(reason, remaining))); return; } @@ -505,7 +556,7 @@ impl LightSync { AbortReason::NoResponses => {} AbortReason::TargetReached => { debug!(target: "sync", "Sync target reached. Going idle"); - *state = SyncState::Idle; + self.set_state(&mut state, SyncState::Idle); return; } } @@ -514,15 +565,15 @@ impl LightSync { self.begin_search(&mut state); } SyncState::AncestorSearch(AncestorSearch::FoundCommon(num, hash)) => { - *state = SyncState::Rounds(SyncRound::begin((num, hash), sync_target)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((num, hash), sync_target))); } SyncState::AncestorSearch(AncestorSearch::Genesis) => { // Same here. let g_hash = chain_info.genesis_hash; - *state = SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target))); } SyncState::Idle => self.begin_search(&mut state), - other => *state = other, // restore displaced state. + other => self.set_state(&mut state, other), // restore displaced state. } } @@ -543,12 +594,13 @@ impl LightSync { } drop(pending_reqs); - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.requests_abandoned(&unfulfilled)), SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(&unfulfilled)), }; + self.set_state(&mut state, next_state); } } @@ -605,34 +657,14 @@ impl LightSync { None }; - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(round) => SyncState::Rounds(round.dispatch_requests(dispatcher)), SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.dispatch_request(dispatcher)), other => other, }; - } - } - - fn is_major_importing_do_wait(&self, wait: bool) -> bool { - const EMPTY_QUEUE: usize = 3; - - if self.client.as_light_client().queue_info().unverified_queue_size > EMPTY_QUEUE { - return true; - } - let mg_state = if wait { - self.state.lock() - } else { - if let Some(mg_state) = self.state.try_lock() { - mg_state - } else { - return false; - } - }; - match *mg_state { - SyncState::Idle => false, - _ => true, + self.set_state(&mut state, next_state); } } } @@ -651,7 +683,8 @@ impl LightSync { pending_reqs: Mutex::new(HashMap::new()), client: client, rng: Mutex::new(OsRng::new()?), - state: Mutex::new(SyncState::Idle), + state: Mutex::new(SyncStateWrapper::idle()), + is_idle: Mutex::new(true), }) } } @@ -666,9 +699,6 @@ pub trait SyncInfo { /// Whether major sync is underway. fn is_major_importing(&self) -> bool; - - /// Whether major sync is underway, skipping some synchronization. - fn is_major_importing_no_sync(&self) -> bool; } impl SyncInfo for LightSync { @@ -681,11 +711,13 @@ impl SyncInfo for LightSync { } fn is_major_importing(&self) -> bool { - self.is_major_importing_do_wait(true) - } + const EMPTY_QUEUE: usize = 3; - fn is_major_importing_no_sync(&self) -> bool { - self.is_major_importing_do_wait(false) + let queue_info = self.client.as_light_client().queue_info(); + let is_verifying = queue_info.unverified_queue_size + queue_info.verified_queue_size > EMPTY_QUEUE; + let is_syncing = !*self.is_idle.lock(); + + is_verifying || is_syncing } } diff --git a/ethcore/sync/src/sync_io.rs b/ethcore/sync/src/sync_io.rs index c7704724c..a5e9f7b2f 100644 --- a/ethcore/sync/src/sync_io.rs +++ b/ethcore/sync/src/sync_io.rs @@ -52,7 +52,7 @@ pub trait SyncIo { fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8; /// Returns if the chain block queue empty fn is_chain_queue_empty(&self) -> bool { - self.chain().queue_info().is_empty() + self.chain().is_queue_empty() } /// Check if the session is expired fn is_expired(&self) -> bool; diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index d75d71ea9..3eac91a0d 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -33,7 +33,7 @@ use ethcore::test_helpers; use sync_io::SyncIo; use io::{IoChannel, IoContext, IoHandler}; use api::WARP_SYNC_PROTOCOL_ID; -use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; +use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET, SyncSupplier}; use SyncConfig; use private_tx::SimplePrivateTxHandler; @@ -271,7 +271,7 @@ impl Peer for EthPeer { fn receive_message(&self, from: PeerId, msg: TestPacket) -> HashSet { let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, Some(from)); - ChainSync::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); + SyncSupplier::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); self.chain.flush(); io.to_disconnect.clone() } @@ -286,10 +286,12 @@ impl Peer for EthPeer { } fn sync_step(&self) { + let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None); self.chain.flush(); - self.sync.write().maintain_peers(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().maintain_sync(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().propagate_new_transactions(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); + self.sync.write().maintain_peers(&mut io); + self.sync.write().maintain_sync(&mut io); + self.sync.write().continue_sync(&mut io); + self.sync.write().propagate_new_transactions(&mut io); } fn restart_sync(&self) { diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 4d11dcf68..7d5e2ca4a 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use api::TransactionStats; +use std::hash::BuildHasher; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use fastmap::H256FastMap; @@ -74,7 +75,7 @@ impl TransactionsStats { } /// Retains only transactions present in given `HashSet`. - pub fn retain(&mut self, hashes: &HashSet) { + pub fn retain(&mut self, hashes: &HashSet) { let to_remove = self.pending_transactions.keys() .filter(|hash| !hashes.contains(hash)) .cloned() diff --git a/json/src/spec/account.rs b/json/src/spec/account.rs index acc6d96b5..9a593db0c 100644 --- a/json/src/spec/account.rs +++ b/json/src/spec/account.rs @@ -23,6 +23,7 @@ use spec::builtin::Builtin; /// Spec account. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Account { /// Builtin contract. pub builtin: Option, diff --git a/json/src/spec/authority_round.rs b/json/src/spec/authority_round.rs index b4fcf4d78..17f70dcdd 100644 --- a/json/src/spec/authority_round.rs +++ b/json/src/spec/authority_round.rs @@ -23,6 +23,7 @@ use super::ValidatorSet; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRoundParams { /// Block duration, in seconds. #[serde(rename="stepDuration")] @@ -71,6 +72,7 @@ pub struct AuthorityRoundParams { /// Authority engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRound { /// Ethash params. pub params: AuthorityRoundParams, diff --git a/json/src/spec/basic_authority.rs b/json/src/spec/basic_authority.rs index 1e5c6b845..d9b873366 100644 --- a/json/src/spec/basic_authority.rs +++ b/json/src/spec/basic_authority.rs @@ -21,6 +21,7 @@ use super::ValidatorSet; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BasicAuthorityParams { /// Block duration. #[serde(rename="durationLimit")] @@ -31,6 +32,7 @@ pub struct BasicAuthorityParams { /// Authority engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BasicAuthority { /// Ethash params. pub params: BasicAuthorityParams, diff --git a/json/src/spec/builtin.rs b/json/src/spec/builtin.rs index 850867d09..df889a377 100644 --- a/json/src/spec/builtin.rs +++ b/json/src/spec/builtin.rs @@ -20,6 +20,7 @@ use uint::Uint; /// Linear pricing. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Linear { /// Base price. pub base: usize, @@ -29,6 +30,7 @@ pub struct Linear { /// Pricing for modular exponentiation. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Modexp { /// Price divisor. pub divisor: usize, @@ -36,6 +38,7 @@ pub struct Modexp { /// Pricing for alt_bn128_pairing. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct AltBn128Pairing { /// Base price. pub base: usize, @@ -45,6 +48,7 @@ pub struct AltBn128Pairing { /// Pricing variants. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub enum Pricing { /// Linear pricing. #[serde(rename="linear")] @@ -59,6 +63,7 @@ pub enum Pricing { /// Spec builtin. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Builtin { /// Builtin name. pub name: String, diff --git a/json/src/spec/engine.rs b/json/src/spec/engine.rs index 020cc1fd0..2026fd25c 100644 --- a/json/src/spec/engine.rs +++ b/json/src/spec/engine.rs @@ -20,6 +20,7 @@ use super::{Ethash, BasicAuthority, AuthorityRound, Tendermint, NullEngine, Inst /// Engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum Engine { /// Null engine. #[serde(rename="null")] @@ -28,6 +29,7 @@ pub enum Engine { #[serde(rename="instantSeal")] InstantSeal(Option), /// Ethash engine. + #[serde(rename = "Ethash")] Ethash(Ethash), /// BasicAuthority engine. #[serde(rename="basicAuthority")] @@ -88,7 +90,6 @@ mod tests { "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", - "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "homesteadTransition" : "0x", "daoHardforkTransition": "0xffffffffffffffff", "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", diff --git a/json/src/spec/ethash.rs b/json/src/spec/ethash.rs index 82cc986b2..f73bce969 100644 --- a/json/src/spec/ethash.rs +++ b/json/src/spec/ethash.rs @@ -23,6 +23,7 @@ use hash::Address; /// Deserializable doppelganger of block rewards for EthashParams #[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum BlockReward { Single(Uint), @@ -31,6 +32,7 @@ pub enum BlockReward { /// Deserializable doppelganger of EthashParams. #[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct EthashParams { /// See main EthashParams docs. #[serde(rename="minimumDifficulty")] @@ -119,6 +121,7 @@ pub struct EthashParams { /// Ethash engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Ethash { /// Ethash params. pub params: EthashParams, diff --git a/json/src/spec/genesis.rs b/json/src/spec/genesis.rs index d8e2ad535..cae2098d3 100644 --- a/json/src/spec/genesis.rs +++ b/json/src/spec/genesis.rs @@ -23,6 +23,7 @@ use spec::Seal; /// Spec genesis. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Genesis { /// Seal. pub seal: Seal, @@ -70,7 +71,6 @@ mod tests { #[test] fn genesis_deserialization() { let s = r#"{ - "nonce": "0x0000000000000042", "difficulty": "0x400000000", "seal": { "ethereum": { diff --git a/json/src/spec/hardcoded_sync.rs b/json/src/spec/hardcoded_sync.rs index 8b00b5413..ab671994d 100644 --- a/json/src/spec/hardcoded_sync.rs +++ b/json/src/spec/hardcoded_sync.rs @@ -21,6 +21,7 @@ use uint::Uint; /// Spec hardcoded sync. #[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct HardcodedSync { /// Hexadecimal of the RLP encoding of the header of the block to start synchronization from. pub header: String, @@ -28,7 +29,7 @@ pub struct HardcodedSync { #[serde(rename="totalDifficulty")] pub total_difficulty: Uint, /// Ordered trie roots of blocks before and including `header`. - #[serde(rename="CHTs")] + #[serde(rename = "CHTs")] pub chts: Vec, } diff --git a/json/src/spec/instant_seal.rs b/json/src/spec/instant_seal.rs index 4621a19dd..671f99c7b 100644 --- a/json/src/spec/instant_seal.rs +++ b/json/src/spec/instant_seal.rs @@ -18,6 +18,7 @@ /// Instant seal engine params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct InstantSealParams { /// Whether to enable millisecond timestamp. #[serde(rename="millisecondTimestamp")] @@ -27,6 +28,7 @@ pub struct InstantSealParams { /// Instant seal engine descriptor. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct InstantSeal { /// Instant seal parameters. pub params: InstantSealParams, diff --git a/json/src/spec/null_engine.rs b/json/src/spec/null_engine.rs index 87827bd5b..061c25aac 100644 --- a/json/src/spec/null_engine.rs +++ b/json/src/spec/null_engine.rs @@ -20,6 +20,7 @@ use uint::Uint; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct NullEngineParams { /// Block reward. #[serde(rename="blockReward")] @@ -28,6 +29,7 @@ pub struct NullEngineParams { /// Null engine descriptor #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct NullEngine { /// Ethash params. pub params: NullEngineParams, diff --git a/json/src/spec/params.rs b/json/src/spec/params.rs index ea0a55d49..801a5f3a5 100644 --- a/json/src/spec/params.rs +++ b/json/src/spec/params.rs @@ -22,6 +22,7 @@ use bytes::Bytes; /// Spec params. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Params { /// Account start nonce, defaults to 0. #[serde(rename="accountStartNonce")] diff --git a/json/src/spec/seal.rs b/json/src/spec/seal.rs index b61d141d6..891e00536 100644 --- a/json/src/spec/seal.rs +++ b/json/src/spec/seal.rs @@ -22,6 +22,7 @@ use bytes::Bytes; /// Ethereum seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Ethereum { /// Seal nonce. pub nonce: H64, @@ -32,6 +33,7 @@ pub struct Ethereum { /// AuthorityRound seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRoundSeal { /// Seal step. pub step: Uint, @@ -41,6 +43,7 @@ pub struct AuthorityRoundSeal { /// Tendermint seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct TendermintSeal { /// Seal round. pub round: Uint, @@ -52,6 +55,7 @@ pub struct TendermintSeal { /// Seal variants. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum Seal { /// Ethereum seal. #[serde(rename="ethereum")] diff --git a/json/src/spec/spec.rs b/json/src/spec/spec.rs index a15d1faf6..e0a869891 100644 --- a/json/src/spec/spec.rs +++ b/json/src/spec/spec.rs @@ -38,6 +38,7 @@ pub enum ForkSpec { /// Spec deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Spec { /// Spec name. pub name: String, @@ -71,6 +72,71 @@ mod tests { use serde_json; use spec::spec::Spec; + #[test] + fn should_error_on_unknown_fields() { + let s = r#"{ + "name": "Morden", + "dataDir": "morden", + "engine": { + "Ethash": { + "params": { + "minimumDifficulty": "0x020000", + "difficultyBoundDivisor": "0x0800", + "durationLimit": "0x0d", + "homesteadTransition" : "0x", + "daoHardforkTransition": "0xffffffffffffffff", + "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", + "daoHardforkAccounts": [] + } + } + }, + "params": { + "accountStartNonce": "0x0100000", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID" : "0x2", + "forkBlock": "0xffffffffffffffff", + "forkCanonHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "gasLimitBoundDivisor": "0x20", + "unknownField": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x00006d6f7264656e" + } + }, + "difficulty": "0x20000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x", + "gasLimit": "0x2fefd8" + }, + "nodes": [ + "enode://b1217cbaa440e35ed471157123fe468e19e8b5ad5bedb4b1fdbcbdab6fb2f5ed3e95dd9c24a22a79fdb2352204cea207df27d92bfd21bfd41545e8b16f637499@104.44.138.37:30303" + ], + "accounts": { + "0000000000000000000000000000000000000001": { "balance": "1", "nonce": "1048576", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "balance": "1", "nonce": "1048576", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "balance": "1", "nonce": "1048576", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "balance": "1", "nonce": "1048576", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "102e61f5d8f9bc71d0ad4a084df4e65e05ce0e1c": { "balance": "1606938044258990275541962092341162602522202993782792835301376", "nonce": "1048576" } + }, + "hardcodedSync": { + "header": "f901f9a0d405da4e66f1445d455195229624e133f5baafe72b5cf7b3c36c12c8146e98b7a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a05fb2b4bfdef7b314451cb138a534d225c922fc0e5fbe25e451142732c3e25c25a088d2ec6b9860aae1a2c3b299f72b6a5d70d7f7ba4722c78f2c49ba96273c2158a007c6fdfa8eea7e86b81f5b0fc0f78f90cc19f4aa60d323151e0cac660199e9a1b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefba82524d84568e932a80a0a0349d8c3df71f1a48a9df7d03fd5f14aeee7d91332c009ecaff0a71ead405bd88ab4e252a7e8c2a23", + "totalDifficulty": "0x400000000", + "CHTs": [ + "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", + "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544" + ] + } + }"#; + let result: Result = serde_json::from_str(s); + assert!(result.is_err()); + } + #[test] fn spec_deserialization() { let s = r#"{ @@ -91,7 +157,6 @@ mod tests { }, "params": { "accountStartNonce": "0x0100000", - "homesteadTransition": "0x789b0", "maximumExtraDataSize": "0x20", "minGasLimit": "0x1388", "networkID" : "0x2", diff --git a/json/src/spec/state.rs b/json/src/spec/state.rs index d15ad540c..6227750e1 100644 --- a/json/src/spec/state.rs +++ b/json/src/spec/state.rs @@ -23,6 +23,7 @@ use spec::{Account, Builtin}; /// Blockchain test state deserializer. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct State(BTreeMap); impl State { diff --git a/json/src/spec/tendermint.rs b/json/src/spec/tendermint.rs index e0a6568aa..da4371531 100644 --- a/json/src/spec/tendermint.rs +++ b/json/src/spec/tendermint.rs @@ -21,6 +21,7 @@ use super::ValidatorSet; /// Tendermint params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct TendermintParams { /// Valid validators. pub validators: ValidatorSet, @@ -43,6 +44,7 @@ pub struct TendermintParams { /// Tendermint engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Tendermint { /// Ethash params. pub params: TendermintParams, diff --git a/json/src/spec/validator_set.rs b/json/src/spec/validator_set.rs index 41fa60961..c754f79a9 100644 --- a/json/src/spec/validator_set.rs +++ b/json/src/spec/validator_set.rs @@ -22,6 +22,7 @@ use hash::Address; /// Different ways of specifying validators. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum ValidatorSet { /// A simple list of authorities. #[serde(rename="list")] diff --git a/miner/src/pool/listener.rs b/miner/src/pool/listener.rs index 5776ba845..f5e76e15d 100644 --- a/miner/src/pool/listener.rs +++ b/miner/src/pool/listener.rs @@ -50,6 +50,10 @@ impl Notifier { /// Notify listeners about all currently pending transactions. pub fn notify(&mut self) { + if self.pending.is_empty() { + return; + } + for l in &self.listeners { (l)(&self.pending); } diff --git a/parity/cli/mod.rs b/parity/cli/mod.rs index 5c3bddf6c..17e261800 100644 --- a/parity/cli/mod.rs +++ b/parity/cli/mod.rs @@ -467,6 +467,10 @@ usage! { "--no-jsonrpc", "Disable the HTTP JSON-RPC API server.", + FLAG flag_jsonrpc_experimental: (bool) = false, or |c: &Config| c.rpc.as_ref()?.experimental_rpcs.clone(), + "--jsonrpc-experimental", + "Enable experimental RPCs. Enable to have access to methods from unfinalised EIPs in all namespaces", + ARG arg_jsonrpc_port: (u16) = 8545u16, or |c: &Config| c.rpc.as_ref()?.port.clone(), "--jsonrpc-port=[PORT]", "Specify the port portion of the HTTP JSON-RPC API server.", @@ -1141,7 +1145,7 @@ struct Operating { no_persistent_txqueue: Option, no_hardcoded_sync: Option, - #[serde(rename="public_node")] + #[serde(rename = "public_node")] _legacy_public_node: Option, } @@ -1173,15 +1177,15 @@ struct PrivateTransactions { struct Ui { path: Option, - #[serde(rename="force")] + #[serde(rename = "force")] _legacy_force: Option, - #[serde(rename="disable")] + #[serde(rename = "disable")] _legacy_disable: Option, - #[serde(rename="port")] + #[serde(rename = "port")] _legacy_port: Option, - #[serde(rename="interface")] + #[serde(rename = "interface")] _legacy_interface: Option, - #[serde(rename="hosts")] + #[serde(rename = "hosts")] _legacy_hosts: Option>, } @@ -1219,6 +1223,7 @@ struct Rpc { server_threads: Option, processing_threads: Option, max_payload: Option, + experimental_rpcs: Option, } #[derive(Default, Debug, PartialEq, Deserialize)] @@ -1244,21 +1249,21 @@ struct Ipc { #[derive(Default, Debug, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] struct Dapps { - #[serde(rename="disable")] + #[serde(rename = "disable")] _legacy_disable: Option, - #[serde(rename="port")] + #[serde(rename = "port")] _legacy_port: Option, - #[serde(rename="interface")] + #[serde(rename = "interface")] _legacy_interface: Option, - #[serde(rename="hosts")] + #[serde(rename = "hosts")] _legacy_hosts: Option>, - #[serde(rename="cors")] + #[serde(rename = "cors")] _legacy_cors: Option, - #[serde(rename="path")] + #[serde(rename = "path")] _legacy_path: Option, - #[serde(rename="user")] + #[serde(rename = "user")] _legacy_user: Option, - #[serde(rename="pass")] + #[serde(rename = "pass")] _legacy_pass: Option, } @@ -1676,6 +1681,7 @@ mod tests { // -- API and Console Options // RPC flag_no_jsonrpc: false, + flag_jsonrpc_experimental: false, arg_jsonrpc_port: 8545u16, arg_jsonrpc_interface: "local".into(), arg_jsonrpc_cors: "null".into(), @@ -1958,6 +1964,7 @@ mod tests { server_threads: None, processing_threads: None, max_payload: None, + experimental_rpcs: None, }), ipc: Some(Ipc { disable: None, diff --git a/parity/configuration.rs b/parity/configuration.rs index 8ec2fa88c..661a8d9b2 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -138,6 +138,7 @@ impl Configuration { let compaction = self.args.arg_db_compaction.parse()?; let warp_sync = !self.args.flag_no_warp; let geth_compatibility = self.args.flag_geth; + let experimental_rpcs = self.args.flag_jsonrpc_experimental; let ipfs_conf = self.ipfs_config(); let secretstore_conf = self.secretstore_config()?; let format = self.format()?; @@ -377,6 +378,7 @@ impl Configuration { warp_sync: warp_sync, warp_barrier: self.args.arg_warp_barrier, geth_compatibility: geth_compatibility, + experimental_rpcs, net_settings: self.network_settings()?, ipfs_conf: ipfs_conf, secretstore_conf: secretstore_conf, @@ -1418,6 +1420,7 @@ mod tests { compaction: Default::default(), vm_type: Default::default(), geth_compatibility: false, + experimental_rpcs: false, net_settings: Default::default(), ipfs_conf: Default::default(), secretstore_conf: Default::default(), diff --git a/parity/informant.rs b/parity/informant.rs index 905978617..8cc37813c 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -184,7 +184,7 @@ impl InformantData for LightNodeInformantData { fn executes_transactions(&self) -> bool { false } fn is_major_importing(&self) -> bool { - self.sync.is_major_importing_no_sync() + self.sync.is_major_importing() } fn report(&self) -> Report { @@ -256,16 +256,13 @@ impl Informant { } pub fn tick(&self) { - let elapsed = self.last_tick.read().elapsed(); - if elapsed < Duration::from_secs(5) { - return; - } + let now = Instant::now(); + let elapsed = now.duration_since(*self.last_tick.read()); let (client_report, full_report) = { let mut last_report = self.last_report.lock(); let full_report = self.target.report(); let diffed = full_report.client_report.clone() - &*last_report; - *last_report = full_report.client_report.clone(); (diffed, full_report) }; @@ -289,7 +286,8 @@ impl Informant { return; } - *self.last_tick.write() = Instant::now(); + *self.last_tick.write() = now; + *self.last_report.lock() = full_report.client_report.clone(); let paint = |c: Style, t: String| match self.with_color && atty::is(atty::Stream::Stdout) { true => format!("{}", c.paint(t)), @@ -306,7 +304,7 @@ impl Informant { format!("{} blk/s {} tx/s {} Mgas/s", paint(Yellow.bold(), format!("{:7.2}", (client_report.blocks_imported * 1000) as f64 / elapsed.as_milliseconds() as f64)), paint(Yellow.bold(), format!("{:6.1}", (client_report.transactions_applied * 1000) as f64 / elapsed.as_milliseconds() as f64)), - paint(Yellow.bold(), format!("{:4}", (client_report.gas_processed / (elapsed.as_milliseconds() * 1000)).low_u64())) + paint(Yellow.bold(), format!("{:6.1}", (client_report.gas_processed / 1000).low_u64() as f64 / elapsed.as_milliseconds() as f64)) ) } else { format!("{} hdr/s", diff --git a/parity/modules.rs b/parity/modules.rs index e12e8ee45..ac84aea5f 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc}; use ethcore::client::BlockChainClient; use sync::{self, AttachedProtocol, SyncConfig, NetworkConfiguration, Params, ConnectionFilter}; @@ -25,12 +25,17 @@ pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler}; pub use ethcore::client::ChainNotify; use ethcore_logger::Config as LogConfig; -pub type SyncModules = (Arc, Arc, Arc); +pub type SyncModules = ( + Arc, + Arc, + Arc, + mpsc::Sender, +); pub fn sync( - sync_cfg: SyncConfig, - net_cfg: NetworkConfiguration, - client: Arc, + config: SyncConfig, + network_config: NetworkConfiguration, + chain: Arc, snapshot_service: Arc, private_tx_handler: Arc, provider: Arc, @@ -39,15 +44,20 @@ pub fn sync( connection_filter: Option>, ) -> Result { let eth_sync = EthSync::new(Params { - config: sync_cfg, - chain: client, - provider: provider, - snapshot_service: snapshot_service, + config, + chain, + provider, + snapshot_service, private_tx_handler, - network_config: net_cfg, - attached_protos: attached_protos, + network_config, + attached_protos, }, connection_filter)?; - Ok((eth_sync.clone() as Arc, eth_sync.clone() as Arc, eth_sync.clone() as Arc)) + Ok(( + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.priority_tasks() + )) } diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index d07b25a8d..9839b70d2 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -228,6 +228,7 @@ pub struct FullDependencies { pub net_service: Arc, pub updater: Arc, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub ws_address: Option, pub fetch: FetchClient, pub executor: Executor, @@ -317,7 +318,7 @@ impl FullDependencies { } }, Api::Personal => { - handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility).to_delegate()); + handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility, self.experimental_rpcs).to_delegate()); }, Api::Signer => { handler.extend_with(SignerClient::new(&self.secret_store, dispatcher.clone(), &self.signer_service, self.executor.clone()).to_delegate()); @@ -438,6 +439,7 @@ pub struct LightDependencies { pub ws_address: Option, pub fetch: FetchClient, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub executor: Executor, pub whisper_rpc: Option<::whisper::RpcFactory>, pub private_tx_service: Option>, @@ -531,7 +533,7 @@ impl LightDependencies { handler.extend_with(EthPubSub::to_delegate(client)); }, Api::Personal => { - handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility).to_delegate()); + handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility, self.experimental_rpcs).to_delegate()); }, Api::Signer => { handler.extend_with(SignerClient::new(&self.secret_store, dispatcher.clone(), &self.signer_service, self.executor.clone()).to_delegate()); diff --git a/parity/run.rs b/parity/run.rs index 3a176057a..14fd58dd1 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use std::any::Any; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Weak, atomic}; use std::time::{Duration, Instant}; use std::thread; @@ -115,6 +115,7 @@ pub struct RunCmd { pub compaction: DatabaseCompactionProfile, pub vm_type: VMType, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub net_settings: NetworkSettings, pub ipfs_conf: ipfs::Configuration, pub secretstore_conf: secretstore::Configuration, @@ -312,6 +313,7 @@ fn execute_light_impl(cmd: RunCmd, logger: Arc) -> Result(cmd: RunCmd, logger: Arc, on_client_rq: // create dirs used by parity cmd.dirs.create_dirs(cmd.acc_conf.unlocked_accounts.len() == 0, cmd.secretstore_conf.enabled)?; - // run in daemon mode - if let Some(pid_file) = cmd.daemon { - daemonize(pid_file)?; - } - //print out running parity environment print_running_environment(&spec.data_dir, &cmd.dirs, &db_dirs); @@ -482,7 +479,6 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: cmd.gas_pricer_conf.to_gas_pricer(fetch.clone(), runtime.executor()), &spec, Some(account_provider.clone()), - )); miner.set_author(cmd.miner_extras.author, None).expect("Fails only if password is Some; password is None; qed"); miner.set_gas_range_target(cmd.miner_extras.gas_range_target); @@ -639,7 +635,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: }; // create sync object - let (sync_provider, manage_network, chain_notify) = modules::sync( + let (sync_provider, manage_network, chain_notify, priority_tasks) = modules::sync( sync_config, net_conf.clone().into(), client.clone(), @@ -653,6 +649,18 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: service.add_notify(chain_notify.clone()); + // Propagate transactions as soon as they are imported. + let tx = ::parking_lot::Mutex::new(priority_tasks); + let is_ready = Arc::new(atomic::AtomicBool::new(true)); + miner.add_transactions_listener(Box::new(move |_hashes| { + // we want to have only one PendingTransactions task in the queue. + if is_ready.compare_and_swap(true, false, atomic::Ordering::SeqCst) { + let task = ::sync::PriorityTask::PropagateTransactions(Instant::now(), is_ready.clone()); + // we ignore error cause it means that we are closing + let _ = tx.lock().send(task); + } + })); + // provider not added to a notification center is effectively disabled // TODO [debris] refactor it later on if cmd.private_tx_enabled { @@ -712,6 +720,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: net_service: manage_network.clone(), updater: updater.clone(), geth_compatibility: cmd.geth_compatibility, + experimental_rpcs: cmd.experimental_rpcs, ws_address: cmd.ws_conf.address(), fetch: fetch.clone(), executor: runtime.executor(), @@ -737,7 +746,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: let secretstore_deps = secretstore::Dependencies { client: client.clone(), sync: sync_provider.clone(), - miner: miner, + miner: miner.clone(), account_provider: account_provider, accounts_passwords: &passwords, }; @@ -798,6 +807,12 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: client.set_exit_handler(on_client_rq); updater.set_exit_handler(on_updater_rq); + // run in daemon mode + if let Some(pid_file) = cmd.daemon { + info!("Running as a daemon process!"); + daemonize(pid_file)?; + } + Ok(RunningClient { inner: RunningClientInner::Full { rpc: rpc_direct, diff --git a/rpc/src/v1/extractors.rs b/rpc/src/v1/extractors.rs index f0433db02..a7c1c40b1 100644 --- a/rpc/src/v1/extractors.rs +++ b/rpc/src/v1/extractors.rs @@ -218,9 +218,10 @@ impl> WsDispatcher { impl> core::Middleware for WsDispatcher { type Future = Either< - core::FutureRpcResult, + core::FutureRpcResult, core::FutureResponse, >; + type CallFuture = core::middleware::NoopCallFuture; fn on_request(&self, request: core::Request, meta: Metadata, process: F) -> Either diff --git a/rpc/src/v1/helpers/errors.rs b/rpc/src/v1/helpers/errors.rs index f177e2515..38279d3b7 100644 --- a/rpc/src/v1/helpers/errors.rs +++ b/rpc/src/v1/helpers/errors.rs @@ -53,6 +53,7 @@ mod codes { pub const FETCH_ERROR: i64 = -32060; pub const NO_LIGHT_PEERS: i64 = -32065; pub const DEPRECATED: i64 = -32070; + pub const EXPERIMENTAL_RPC: i64 = -32071; } pub fn unimplemented(details: Option) -> Error { @@ -500,3 +501,15 @@ pub fn on_demand_others(err: &OnDemandError) -> Error { } } +/// Returns a descriptive error in case experimental RPCs are not enabled. +pub fn require_experimental(allow_experimental_rpcs: bool, eip: &str) -> Result<(), Error> { + if allow_experimental_rpcs { + Ok(()) + } else { + Err(Error { + code: ErrorCode::ServerError(codes::EXPERIMENTAL_RPC), + message: format!("This method is not part of the official RPC API yet (EIP-{}). Run with `--jsonrpc-experimental` to enable it.", eip), + data: Some(Value::String(format!("See EIP: https://eips.ethereum.org/EIPS/eip-{}", eip))), + }) + } +} diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 8982b73e1..e973f7272 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -434,7 +434,7 @@ impl Parity for ParityClient where BlockNumber::Earliest => BlockId::Earliest, BlockNumber::Latest => BlockId::Latest, }; - let receipts = try_bf!(self.client.block_receipts(id).ok_or_else(errors::unknown_block)); + let receipts = try_bf!(self.client.localized_block_receipts(id).ok_or_else(errors::unknown_block)); Box::new(future::ok(receipts.into_iter().map(Into::into).collect())) } diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index 4da0c1023..e42a91b1f 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -47,15 +47,22 @@ pub struct PersonalClient { accounts: Arc, dispatcher: D, allow_perm_unlock: bool, + allow_experimental_rpcs: bool, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(accounts: &Arc, dispatcher: D, allow_perm_unlock: bool) -> Self { + pub fn new( + accounts: &Arc, + dispatcher: D, + allow_perm_unlock: bool, + allow_experimental_rpcs: bool, + ) -> Self { PersonalClient { accounts: accounts.clone(), dispatcher, allow_perm_unlock, + allow_experimental_rpcs, } } } @@ -154,6 +161,8 @@ impl Personal for PersonalClient { } fn sign_191(&self, version: EIP191Version, data: Value, account: RpcH160, password: String) -> BoxFuture { + try_bf!(errors::require_experimental(self.allow_experimental_rpcs, "191")); + let data = try_bf!(eip191::hash_message(version, data)); let dispatcher = self.dispatcher.clone(); let accounts = self.accounts.clone(); @@ -174,6 +183,8 @@ impl Personal for PersonalClient { } fn sign_typed_data(&self, typed_data: EIP712, account: RpcH160, password: String) -> BoxFuture { + try_bf!(errors::require_experimental(self.allow_experimental_rpcs, "712")); + let data = match hash_structured_data(typed_data) { Ok(d) => d, Err(err) => return Box::new(future::err(errors::invalid_call_data(err.kind()))), diff --git a/rpc/src/v1/informant.rs b/rpc/src/v1/informant.rs index 3dab54843..dfa20ee46 100644 --- a/rpc/src/v1/informant.rs +++ b/rpc/src/v1/informant.rs @@ -205,6 +205,7 @@ impl Middleware { impl core::Middleware for Middleware { type Future = core::FutureResponse; + type CallFuture = core::middleware::NoopCallFuture; fn on_request(&self, request: core::Request, meta: M, process: F) -> Either where F: FnOnce(core::Request, M) -> X, diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 39235c690..1a8f3248a 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -243,6 +243,7 @@ const TRANSACTION_COUNT_SPEC: &'static [u8] = br#"{ "params": { "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", + "blockReward": "0x4563918244F40000", "durationLimit": "0x0d", "homesteadTransition": "0xffffffffffffffff", "daoHardforkTransition": "0xffffffffffffffff", @@ -253,7 +254,6 @@ const TRANSACTION_COUNT_SPEC: &'static [u8] = br#"{ }, "params": { "gasLimitBoundDivisor": "0x0400", - "blockReward": "0x4563918244F40000", "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "accountStartNonce": "0x00", "maximumExtraDataSize": "0x20", @@ -292,6 +292,7 @@ const POSITIVE_NONCE_SPEC: &'static [u8] = br#"{ "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", + "blockReward": "0x4563918244F40000", "homesteadTransition": "0xffffffffffffffff", "daoHardforkTransition": "0xffffffffffffffff", "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", @@ -301,7 +302,6 @@ const POSITIVE_NONCE_SPEC: &'static [u8] = br#"{ }, "params": { "gasLimitBoundDivisor": "0x0400", - "blockReward": "0x4563918244F40000", "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "accountStartNonce": "0x0100", "maximumExtraDataSize": "0x20", diff --git a/rpc/src/v1/tests/mocked/personal.rs b/rpc/src/v1/tests/mocked/personal.rs index 5098d807d..143b19fa8 100644 --- a/rpc/src/v1/tests/mocked/personal.rs +++ b/rpc/src/v1/tests/mocked/personal.rs @@ -57,6 +57,16 @@ fn miner_service() -> Arc { } fn setup() -> PersonalTester { + setup_with(Config { + allow_experimental_rpcs: true + }) +} + +struct Config { + pub allow_experimental_rpcs: bool, +} + +fn setup_with(c: Config) -> PersonalTester { let runtime = Runtime::with_thread_count(1); let accounts = accounts_provider(); let client = blockchain_client(); @@ -64,7 +74,7 @@ fn setup() -> PersonalTester { let reservations = Arc::new(Mutex::new(nonce::Reservations::new(runtime.executor()))); let dispatcher = FullDispatcher::new(client, miner.clone(), reservations, 50); - let personal = PersonalClient::new(&accounts, dispatcher, false); + let personal = PersonalClient::new(&accounts, dispatcher, false, c.allow_experimental_rpcs); let mut io = IoHandler::default(); io.extend_with(personal.to_delegate()); @@ -418,3 +428,109 @@ fn sign_eip191_structured_data() { let response = tester.io.handle_request_sync(&request).unwrap(); assert_eq!(response, expected) } + +#[test] +fn sign_structured_data() { + let tester = setup(); + let secret: Secret = keccak("cow").into(); + let address = tester.accounts.insert_account(secret, &"lol".into()).unwrap(); + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_signTypedData", + "params": [ + { + "primaryType": "Mail", + "domain": { + "name": "Ether Mail", + "version": "1", + "chainId": "0x1", + "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC" + }, + "message": { + "from": { + "name": "Cow", + "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826" + }, + "to": { + "name": "Bob", + "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + }, + "contents": "Hello, Bob!" + }, + "types": { + "EIP712Domain": [ + { "name": "name", "type": "string" }, + { "name": "version", "type": "string" }, + { "name": "chainId", "type": "uint256" }, + { "name": "verifyingContract", "type": "address" } + ], + "Person": [ + { "name": "name", "type": "string" }, + { "name": "wallet", "type": "address" } + ], + "Mail": [ + { "name": "from", "type": "Person" }, + { "name": "to", "type": "Person" }, + { "name": "contents", "type": "string" } + ] + } + }, + ""#.to_owned() + &format!("0x{:x}", address) + r#"", + "lol" + ], + "id": 1 + }"#; + let expected = r#"{"jsonrpc":"2.0","result":"0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c","id":1}"#; + let response = tester.io.handle_request_sync(&request).unwrap(); + assert_eq!(response, expected) +} + +#[test] +fn should_disable_experimental_apis() { + // given + let tester = setup_with(Config { + allow_experimental_rpcs: false, + }); + + // when + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_sign191", + "params": [ + "0x01", + {}, + "0x1234567891234567891234567891234567891234", + "lol" + ], + "id": 1 + }"#; + let r1 = tester.io.handle_request_sync(&request).unwrap(); + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_signTypedData", + "params": [ + { + "types": {}, + "message": {}, + "domain": { + "name": "", + "version": "1", + "chainId": "0x1", + "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC" + }, + "primaryType": "" + }, + "0x1234567891234567891234567891234678912344", + "lol" + ], + "id": 1 + }"#; + let r2 = tester.io.handle_request_sync(&request).unwrap(); + + // then + let expected = r#"{"jsonrpc":"2.0","error":{"code":-32071,"message":"This method is not part of the official RPC API yet (EIP-191). Run with `--jsonrpc-experimental` to enable it.","data":"See EIP: https://eips.ethereum.org/EIPS/eip-191"},"id":1}"#; + assert_eq!(r1, expected); + + let expected = r#"{"jsonrpc":"2.0","error":{"code":-32071,"message":"This method is not part of the official RPC API yet (EIP-712). Run with `--jsonrpc-experimental` to enable it.","data":"See EIP: https://eips.ethereum.org/EIPS/eip-712"},"id":1}"#; + assert_eq!(r2, expected); +} diff --git a/scripts/docker/hub/Dockerfile b/scripts/docker/hub/Dockerfile index 1ac92e71b..e9813754b 100644 --- a/scripts/docker/hub/Dockerfile +++ b/scripts/docker/hub/Dockerfile @@ -18,7 +18,6 @@ RUN rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* RUN groupadd -g 1000 parity \ && useradd -m -u 1000 -g parity -s /bin/sh parity -USER parity WORKDIR /home/parity @@ -33,6 +32,9 @@ RUN chmod +x ./entrypoint.sh COPY scripts/docker/hub/check_sync.sh /check_sync.sh +# switch to user parity here +USER parity + # setup ENTRYPOINT EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp ENTRYPOINT ["./entrypoint.sh"] diff --git a/scripts/gitlab/docs-jsonrpc.sh b/scripts/gitlab/publish-docs.sh similarity index 100% rename from scripts/gitlab/docs-jsonrpc.sh rename to scripts/gitlab/publish-docs.sh diff --git a/util/fastmap/src/lib.rs b/util/fastmap/src/lib.rs index 135ce54ba..65dd9dfb4 100644 --- a/util/fastmap/src/lib.rs +++ b/util/fastmap/src/lib.rs @@ -21,11 +21,13 @@ extern crate plain_hasher; use ethereum_types::H256; use std::hash; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use plain_hasher::PlainHasher; /// Specialized version of `HashMap` with H256 keys and fast hashing function. pub type H256FastMap = HashMap>; +/// Specialized version of HashSet with H256 values and fast hashing function. +pub type H256FastSet = HashSet>; #[cfg(test)] mod tests { @@ -36,4 +38,4 @@ mod tests { let mut h = H256FastMap::default(); h.insert(H256::from(123), "abc"); } -} \ No newline at end of file +} diff --git a/util/version/Cargo.toml b/util/version/Cargo.toml index 854d04c85..f46a0d54d 100644 --- a/util/version/Cargo.toml +++ b/util/version/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "parity-version" # NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION) -version = "2.2.1" +version = "2.2.2" authors = ["Parity Technologies "] build = "build.rs"