diff --git a/Cargo.lock b/Cargo.lock index 20c11bda7..e2108d186 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -548,6 +548,7 @@ dependencies = [ "bigint 4.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "plain_hasher 0.1.0", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -935,6 +936,7 @@ dependencies = [ "docopt 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.8.0", "ethcore-util 1.8.0", + "ethjson 0.1.0", "evm 0.1.0", "panic_hook 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1613,6 +1615,34 @@ dependencies = [ "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "node-filter" +version = "1.8.0" +dependencies = [ + "ethcore 1.8.0", + "ethcore-io 1.8.0", + "ethcore-network 1.8.0", + "ethcore-util 1.8.0", + "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "native-contracts 0.1.0", +] + +[[package]] +name = "node-health" +version = "0.1.0" +dependencies = [ + "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "ntp 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-reactor 0.1.0", + "parking_lot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nodrop" version = "0.1.9" @@ -1821,6 +1851,8 @@ dependencies = [ "isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "node-filter 1.8.0", + "node-health 0.1.0", "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", @@ -1862,7 +1894,6 @@ dependencies = [ "ethcore-util 1.8.0", "fetch 0.1.0", "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "jsonrpc-http-server 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", @@ -1870,7 +1901,7 @@ dependencies = [ "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ntp 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "node-health 0.1.0", "parity-dapps-glue 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.8.0", "parity-reactor 0.1.0", @@ -1985,6 +2016,7 @@ dependencies = [ "jsonrpc-ws-server 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "multihash 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "node-health 0.1.0", "order-stat 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "parity-reactor 0.1.0", "parity-updater 1.8.0", @@ -2058,7 +2090,7 @@ dependencies = [ [[package]] name = "parity-ui-precompiled" version = "1.4.0" -source = "git+https://github.com/paritytech/js-precompiled.git#416ced84c23b1a776d53ee4a3023eb4eb4736cf8" +source = "git+https://github.com/paritytech/js-precompiled.git#d809723e58bcb36c0f8d2eca5ca94abbb3690544" dependencies = [ "parity-dapps-glue 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2197,6 +2229,13 @@ name = "pkg-config" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "plain_hasher" +version = "0.1.0" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "podio" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 3adaf62d4..06abd7b31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,9 @@ ethcore-light = { path = "ethcore/light" } ethcore-logger = { path = "logger" } ethcore-stratum = { path = "stratum" } ethcore-network = { path = "util/network" } +node-filter = { path = "ethcore/node_filter" } ethkey = { path = "ethkey" } +node-health = { path = "dapps/node-health" } rlp = { path = "util/rlp" } rpc-cli = { path = "rpc_cli" } parity-hash-fetch = { path = "hash-fetch" } @@ -110,4 +112,4 @@ lto = false panic = "abort" [workspace] -members = ["ethstore/cli", "ethkey/cli", "evmbin", "whisper", "chainspec"] +members = ["ethstore/cli", "ethkey/cli", "evmbin", "whisper", "chainspec", "dapps/node-health"] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 143bbe30f..b1751c616 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -9,15 +9,12 @@ authors = ["Parity Technologies "] [dependencies] base32 = "0.3" -env_logger = "0.4" futures = "0.1" -futures-cpupool = "0.1" linked-hash-map = "0.3" log = "0.3" parity-dapps-glue = "1.7" mime = "0.2" mime_guess = "1.6.1" -ntp = "0.2.0" rand = "0.3" rustc-hex = "1.0" serde = "1.0" @@ -32,15 +29,19 @@ itertools = "0.5" jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } -ethcore-devtools = { path = "../devtools" } ethcore-util = { path = "../util" } fetch = { path = "../util/fetch" } +node-health = { path = "./node-health" } parity-hash-fetch = { path = "../hash-fetch" } parity-reactor = { path = "../util/reactor" } parity-ui = { path = "./ui" } clippy = { version = "0.0.103", optional = true} +[dev-dependencies] +env_logger = "0.4" +ethcore-devtools = { path = "../devtools" } + [features] dev = ["clippy", "ethcore-util/dev"] diff --git a/dapps/node-health/Cargo.toml b/dapps/node-health/Cargo.toml new file mode 100644 index 000000000..ff13e8593 --- /dev/null +++ b/dapps/node-health/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "node-health" +description = "Node's health status" +version = "0.1.0" +license = "GPL-3.0" +authors = ["Parity Technologies "] + +[dependencies] +futures = "0.1" +futures-cpupool = "0.1" +log = "0.3" +ntp = "0.2.0" +parking_lot = "0.4" +serde = "1.0" +serde_derive = "1.0" +time = "0.1.35" + +parity-reactor = { path = "../../util/reactor" } diff --git a/dapps/node-health/src/health.rs b/dapps/node-health/src/health.rs new file mode 100644 index 000000000..3b3563d6b --- /dev/null +++ b/dapps/node-health/src/health.rs @@ -0,0 +1,122 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Reporting node's health. + +use std::sync::Arc; +use std::time; +use futures::{Future, BoxFuture}; +use futures::sync::oneshot; +use types::{HealthInfo, HealthStatus, Health}; +use time::{TimeChecker, MAX_DRIFT}; +use parity_reactor::Remote; +use parking_lot::Mutex; +use {SyncStatus}; + +const TIMEOUT_SECS: u64 = 5; +const PROOF: &str = "Only one closure is invoked."; + +/// A struct enabling you to query for node's health. +#[derive(Debug, Clone)] +pub struct NodeHealth { + sync_status: Arc, + time: TimeChecker, + remote: Remote, +} + +impl NodeHealth { + /// Creates new `NodeHealth`. + pub fn new(sync_status: Arc, time: TimeChecker, remote: Remote) -> Self { + NodeHealth { sync_status, time, remote, } + } + + /// Query latest health report. + pub fn health(&self) -> BoxFuture { + trace!(target: "dapps", "Checking node health."); + // Check timediff + let sync_status = self.sync_status.clone(); + let time = self.time.time_drift(); + let (tx, rx) = oneshot::channel(); + let tx = Arc::new(Mutex::new(Some(tx))); + let tx2 = tx.clone(); + self.remote.spawn_with_timeout( + move || time.then(move |result| { + let _ = tx.lock().take().expect(PROOF).send(Ok(result)); + Ok(()) + }), + time::Duration::from_secs(TIMEOUT_SECS), + move || { + let _ = tx2.lock().take().expect(PROOF).send(Err(())); + }, + ); + + rx.map_err(|err| { + warn!(target: "dapps", "Health request cancelled: {:?}", err); + }).and_then(move |time| { + // Check peers + let peers = { + let (connected, max) = sync_status.peers(); + let (status, message) = match connected { + 0 => { + (HealthStatus::Bad, "You are not connected to any peers. There is most likely some network issue. Fix connectivity.".into()) + }, + 1 => (HealthStatus::NeedsAttention, "You are connected to only one peer. Your node might not be reliable. Check your network connection.".into()), + _ => (HealthStatus::Ok, "".into()), + }; + HealthInfo { status, message, details: (connected, max) } + }; + + // Check sync + let sync = { + let is_syncing = sync_status.is_major_importing(); + let (status, message) = if is_syncing { + (HealthStatus::NeedsAttention, "Your node is still syncing, the values you see might be outdated. Wait until it's fully synced.".into()) + } else { + (HealthStatus::Ok, "".into()) + }; + HealthInfo { status, message, details: is_syncing } + }; + + // Check time + let time = { + let (status, message, details) = match time { + Ok(Ok(diff)) if diff < MAX_DRIFT && diff > -MAX_DRIFT => { + (HealthStatus::Ok, "".into(), diff) + }, + Ok(Ok(diff)) => { + (HealthStatus::Bad, format!( + "Your clock is not in sync. Detected difference is too big for the protocol to work: {}ms. Synchronize your clock.", + diff, + ), diff) + }, + Ok(Err(err)) => { + (HealthStatus::NeedsAttention, format!( + "Unable to reach time API: {}. Make sure that your clock is synchronized.", + err, + ), 0) + }, + Err(_) => { + (HealthStatus::NeedsAttention, "Time API request timed out. Make sure that the clock is synchronized.".into(), 0) + }, + }; + + HealthInfo { status, message, details, } + }; + + Ok(Health { peers, sync, time}) + }).boxed() + } +} diff --git a/dapps/node-health/src/lib.rs b/dapps/node-health/src/lib.rs new file mode 100644 index 000000000..b0eb133ee --- /dev/null +++ b/dapps/node-health/src/lib.rs @@ -0,0 +1,49 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Node Health status reporting. + +#![warn(missing_docs)] + +extern crate futures; +extern crate futures_cpupool; +extern crate ntp; +extern crate time as time_crate; +extern crate parity_reactor; +extern crate parking_lot; + +#[macro_use] +extern crate log; +#[macro_use] +extern crate serde_derive; + +mod health; +mod time; +mod types; + +pub use futures_cpupool::CpuPool; +pub use health::NodeHealth; +pub use types::{Health, HealthInfo, HealthStatus}; +pub use time::{TimeChecker, Error}; + +/// Indicates sync status +pub trait SyncStatus: ::std::fmt::Debug + Send + Sync { + /// Returns true if there is a major sync happening. + fn is_major_importing(&self) -> bool; + + /// Returns number of connected and ideal peers. + fn peers(&self) -> (usize, usize); +} diff --git a/dapps/src/api/time.rs b/dapps/node-health/src/time.rs similarity index 98% rename from dapps/src/api/time.rs rename to dapps/node-health/src/time.rs index 06b9cee7f..05c48ce47 100644 --- a/dapps/src/api/time.rs +++ b/dapps/node-health/src/time.rs @@ -41,8 +41,8 @@ use futures::{self, Future, BoxFuture}; use futures::future::{self, IntoFuture}; use futures_cpupool::{CpuPool, CpuFuture}; use ntp; -use time::{Duration, Timespec}; -use util::RwLock; +use parking_lot::RwLock; +use time_crate::{Duration, Timespec}; /// Time checker error. #[derive(Debug, Clone, PartialEq)] @@ -164,7 +164,7 @@ impl Ntp for SimpleNtp { match ntp::request(&server.address) { Ok(packet) => { - let dest_time = ::time::now_utc().to_timespec(); + let dest_time = ::time_crate::now_utc().to_timespec(); let orig_time = Timespec::from(packet.orig_time); let recv_time = Timespec::from(packet.recv_time); let transmit_time = Timespec::from(packet.transmit_time); @@ -293,7 +293,7 @@ mod tests { use time::Duration; use futures::{future, Future}; use super::{Ntp, TimeChecker, Error}; - use util::RwLock; + use parking_lot::RwLock; #[derive(Clone)] struct FakeNtp(RefCell>, Cell); diff --git a/dapps/node-health/src/types.rs b/dapps/node-health/src/types.rs new file mode 100644 index 000000000..ae883a626 --- /dev/null +++ b/dapps/node-health/src/types.rs @@ -0,0 +1,57 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Base health types. + +/// Health API endpoint status. +#[derive(Debug, PartialEq, Serialize)] +pub enum HealthStatus { + /// Everything's OK. + #[serde(rename = "ok")] + Ok, + /// Node health need attention + /// (the issue is not critical, but may need investigation) + #[serde(rename = "needsAttention")] + NeedsAttention, + /// There is something bad detected with the node. + #[serde(rename = "bad")] + Bad, +} + +/// Represents a single check in node health. +/// Cointains the status of that check and apropriate message and details. +#[derive(Debug, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct HealthInfo { + /// Check status. + pub status: HealthStatus, + /// Human-readable message. + pub message: String, + /// Technical details of the check. + pub details: T, +} + +/// Node Health status. +#[derive(Debug, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Health { + /// Status of peers. + pub peers: HealthInfo<(usize, usize)>, + /// Sync status. + pub sync: HealthInfo, + /// Time diff info. + pub time: HealthInfo, +} diff --git a/dapps/src/api/api.rs b/dapps/src/api/api.rs index 7bd7fa049..03ba859f8 100644 --- a/dapps/src/api/api.rs +++ b/dapps/src/api/api.rs @@ -21,32 +21,28 @@ use hyper::method::Method; use hyper::status::StatusCode; use api::{response, types}; -use api::time::{TimeChecker, MAX_DRIFT}; use apps::fetcher::Fetcher; use handlers::{self, extract_url}; use endpoint::{Endpoint, Handler, EndpointPath}; +use node_health::{NodeHealth, HealthStatus, Health}; use parity_reactor::Remote; -use {SyncStatus}; #[derive(Clone)] pub struct RestApi { fetcher: Arc, - sync_status: Arc, - time: TimeChecker, + health: NodeHealth, remote: Remote, } impl RestApi { pub fn new( fetcher: Arc, - sync_status: Arc, - time: TimeChecker, + health: NodeHealth, remote: Remote, ) -> Box { Box::new(RestApi { fetcher, - sync_status, - time, + health, remote, }) } @@ -90,68 +86,23 @@ impl RestApiRouter { } fn health(&self, control: Control) -> Box { - use self::types::{HealthInfo, HealthStatus, Health}; - - trace!(target: "dapps", "Checking node health."); - // Check timediff - let sync_status = self.api.sync_status.clone(); - let map = move |time| { - // Check peers - let peers = { - let (connected, max) = sync_status.peers(); - let (status, message) = match connected { - 0 => { - (HealthStatus::Bad, "You are not connected to any peers. There is most likely some network issue. Fix connectivity.".into()) - }, - 1 => (HealthStatus::NeedsAttention, "You are connected to only one peer. Your node might not be reliable. Check your network connection.".into()), - _ => (HealthStatus::Ok, "".into()), - }; - HealthInfo { status, message, details: (connected, max) } + let map = move |health: Result, ()>| { + let status = match health { + Ok(Ok(ref health)) => { + if [&health.peers.status, &health.sync.status].iter().any(|x| *x != &HealthStatus::Ok) { + StatusCode::PreconditionFailed // HTTP 412 + } else { + StatusCode::Ok // HTTP 200 + } + }, + _ => StatusCode::ServiceUnavailable, // HTTP 503 }; - // Check sync - let sync = { - let is_syncing = sync_status.is_major_importing(); - let (status, message) = if is_syncing { - (HealthStatus::NeedsAttention, "Your node is still syncing, the values you see might be outdated. Wait until it's fully synced.".into()) - } else { - (HealthStatus::Ok, "".into()) - }; - HealthInfo { status, message, details: is_syncing } - }; - - // Check time - let time = { - let (status, message, details) = match time { - Ok(Ok(diff)) if diff < MAX_DRIFT && diff > -MAX_DRIFT => { - (HealthStatus::Ok, "".into(), diff) - }, - Ok(Ok(diff)) => { - (HealthStatus::Bad, format!( - "Your clock is not in sync. Detected difference is too big for the protocol to work: {}ms. Synchronize your clock.", - diff, - ), diff) - }, - Ok(Err(err)) => { - (HealthStatus::NeedsAttention, format!( - "Unable to reach time API: {}. Make sure that your clock is synchronized.", - err, - ), 0) - }, - Err(_) => { - (HealthStatus::NeedsAttention, "Time API request timed out. Make sure that the clock is synchronized.".into(), 0) - }, - }; - - HealthInfo { status, message, details, } - }; - - response::as_json(StatusCode::Ok, &Health { peers, sync, time }) + response::as_json(status, &health) }; - - let time = self.api.time.time_drift(); + let health = self.api.health.health(); let remote = self.api.remote.clone(); - Box::new(handlers::AsyncHandler::new(time, map, remote, control)) + Box::new(handlers::AsyncHandler::new(health, map, remote, control)) } } diff --git a/dapps/src/api/mod.rs b/dapps/src/api/mod.rs index 59c634399..4ffb9f791 100644 --- a/dapps/src/api/mod.rs +++ b/dapps/src/api/mod.rs @@ -18,8 +18,6 @@ mod api; mod response; -mod time; mod types; pub use self::api::RestApi; -pub use self::time::TimeChecker; diff --git a/dapps/src/api/types.rs b/dapps/src/api/types.rs index a61964143..6beca3b58 100644 --- a/dapps/src/api/types.rs +++ b/dapps/src/api/types.rs @@ -25,43 +25,3 @@ pub struct ApiError { /// More technical error details. pub detail: String, } - -/// Health API endpoint status. -#[derive(Debug, PartialEq, Serialize)] -pub enum HealthStatus { - /// Everything's OK. - #[serde(rename = "ok")] - Ok, - /// Node health need attention - /// (the issue is not critical, but may need investigation) - #[serde(rename = "needsAttention")] - NeedsAttention, - /// There is something bad detected with the node. - #[serde(rename = "bad")] - Bad -} - -/// Represents a single check in node health. -/// Cointains the status of that check and apropriate message and details. -#[derive(Debug, PartialEq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct HealthInfo { - /// Check status. - pub status: HealthStatus, - /// Human-readable message. - pub message: String, - /// Technical details of the check. - pub details: T, -} - -/// Node Health status. -#[derive(Debug, PartialEq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct Health { - /// Status of peers. - pub peers: HealthInfo<(usize, usize)>, - /// Sync status. - pub sync: HealthInfo, - /// Time diff info. - pub time: HealthInfo, -} diff --git a/dapps/src/apps/fetcher/mod.rs b/dapps/src/apps/fetcher/mod.rs index 1fdf2f697..5b91da1a3 100644 --- a/dapps/src/apps/fetcher/mod.rs +++ b/dapps/src/apps/fetcher/mod.rs @@ -281,6 +281,7 @@ mod tests { } } + #[derive(Debug)] struct FakeSync(bool); impl SyncStatus for FakeSync { fn is_major_importing(&self) -> bool { self.0 } diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index 135f0bb36..073db5121 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -21,11 +21,9 @@ extern crate base32; extern crate futures; -extern crate futures_cpupool; extern crate itertools; extern crate linked_hash_map; extern crate mime_guess; -extern crate ntp; extern crate rand; extern crate rustc_hex; extern crate serde; @@ -40,6 +38,7 @@ extern crate jsonrpc_http_server; extern crate ethcore_util as util; extern crate fetch; +extern crate node_health; extern crate parity_dapps_glue as parity_dapps; extern crate parity_hash_fetch as hash_fetch; extern crate parity_reactor; @@ -57,7 +56,6 @@ extern crate ethcore_devtools as devtools; #[cfg(test)] extern crate env_logger; - mod endpoint; mod apps; mod page; @@ -79,19 +77,12 @@ use util::RwLock; use jsonrpc_http_server::{self as http, hyper, Origin}; use fetch::Fetch; -use futures_cpupool::CpuPool; +use node_health::NodeHealth; use parity_reactor::Remote; pub use hash_fetch::urlhint::ContractClient; +pub use node_health::SyncStatus; -/// Indicates sync status -pub trait SyncStatus: Send + Sync { - /// Returns true if there is a major sync happening. - fn is_major_importing(&self) -> bool; - - /// Returns number of connected and ideal peers. - fn peers(&self) -> (usize, usize); -} /// Validates Web Proxy tokens pub trait WebProxyTokens: Send + Sync { @@ -156,8 +147,7 @@ impl Middleware { /// Creates new middleware for UI server. pub fn ui( - ntp_servers: &[String], - pool: CpuPool, + health: NodeHealth, remote: Remote, dapps_domain: &str, registrar: Arc, @@ -172,11 +162,9 @@ impl Middleware { ).embeddable_on(None).allow_dapps(false)); let special = { let mut special = special_endpoints( - ntp_servers, - pool, + health, content_fetcher.clone(), remote.clone(), - sync_status.clone(), ); special.insert(router::SpecialEndpoint::Home, Some(apps::ui())); special @@ -197,8 +185,7 @@ impl Middleware { /// Creates new Dapps server middleware. pub fn dapps( - ntp_servers: &[String], - pool: CpuPool, + health: NodeHealth, remote: Remote, ui_address: Option<(String, u16)>, extra_embed_on: Vec<(String, u16)>, @@ -236,11 +223,9 @@ impl Middleware { let special = { let mut special = special_endpoints( - ntp_servers, - pool, + health, content_fetcher.clone(), remote.clone(), - sync_status, ); special.insert( router::SpecialEndpoint::Home, @@ -270,20 +255,17 @@ impl http::RequestMiddleware for Middleware { } } -fn special_endpoints>( - ntp_servers: &[T], - pool: CpuPool, +fn special_endpoints( + health: NodeHealth, content_fetcher: Arc, remote: Remote, - sync_status: Arc, ) -> HashMap>> { let mut special = HashMap::new(); special.insert(router::SpecialEndpoint::Rpc, None); special.insert(router::SpecialEndpoint::Utils, Some(apps::utils())); special.insert(router::SpecialEndpoint::Api, Some(api::RestApi::new( content_fetcher, - sync_status, - api::TimeChecker::new(ntp_servers, pool), + health, remote, ))); special diff --git a/dapps/src/tests/helpers/mod.rs b/dapps/src/tests/helpers/mod.rs index 30d3ba8f9..6f4652351 100644 --- a/dapps/src/tests/helpers/mod.rs +++ b/dapps/src/tests/helpers/mod.rs @@ -26,7 +26,7 @@ use jsonrpc_http_server::{self as http, Host, DomainsValidation}; use devtools::http_client; use hash_fetch::urlhint::ContractClient; use fetch::{Fetch, Client as FetchClient}; -use futures_cpupool::CpuPool; +use node_health::{NodeHealth, TimeChecker, CpuPool}; use parity_reactor::Remote; use {Middleware, SyncStatus, WebProxyTokens}; @@ -39,6 +39,7 @@ use self::fetch::FakeFetch; const SIGNER_PORT: u16 = 18180; +#[derive(Debug)] struct FakeSync(bool); impl SyncStatus for FakeSync { fn is_major_importing(&self) -> bool { self.0 } @@ -254,9 +255,13 @@ impl Server { remote: Remote, fetch: F, ) -> Result { + let health = NodeHealth::new( + sync_status.clone(), + TimeChecker::new::(&[], CpuPool::new(1)), + remote.clone(), + ); let middleware = Middleware::dapps( - &["0.pool.ntp.org:123".into(), "1.pool.ntp.org:123".into()], - CpuPool::new(4), + health, remote, signer_address, vec![], diff --git a/ethcore/evm/src/interpreter/memory.rs b/ethcore/evm/src/interpreter/memory.rs index a13c99a82..9aa9babc7 100644 --- a/ethcore/evm/src/interpreter/memory.rs +++ b/ethcore/evm/src/interpreter/memory.rs @@ -44,7 +44,7 @@ pub trait Memory { } /// Checks whether offset and size is valid memory range -fn is_valid_range(off: usize, size: usize) -> bool { +pub fn is_valid_range(off: usize, size: usize) -> bool { // When size is zero we haven't actually expanded the memory let overflow = off.overflowing_add(size).1; size > 0 && !overflow diff --git a/ethcore/evm/src/interpreter/mod.rs b/ethcore/evm/src/interpreter/mod.rs index 3069ab2fe..cb8fe3a52 100644 --- a/ethcore/evm/src/interpreter/mod.rs +++ b/ethcore/evm/src/interpreter/mod.rs @@ -168,7 +168,12 @@ impl vm::Vm for Interpreter { } if do_trace { - ext.trace_executed(gasometer.current_gas.as_u256(), stack.peek_top(info.ret), mem_written.map(|(o, s)| (o, &(self.mem[o..(o + s)]))), store_written); + ext.trace_executed( + gasometer.current_gas.as_u256(), + stack.peek_top(info.ret), + mem_written.map(|(o, s)| (o, &(self.mem[o..o+s]))), + store_written, + ); } // Advance @@ -252,14 +257,20 @@ impl Interpreter { instruction: Instruction, stack: &Stack ) -> Option<(usize, usize)> { - match instruction { - instructions::MSTORE | instructions::MLOAD => Some((stack.peek(0).low_u64() as usize, 32)), - instructions::MSTORE8 => Some((stack.peek(0).low_u64() as usize, 1)), - instructions::CALLDATACOPY | instructions::CODECOPY | instructions::RETURNDATACOPY => Some((stack.peek(0).low_u64() as usize, stack.peek(2).low_u64() as usize)), - instructions::EXTCODECOPY => Some((stack.peek(1).low_u64() as usize, stack.peek(3).low_u64() as usize)), - instructions::CALL | instructions::CALLCODE => Some((stack.peek(5).low_u64() as usize, stack.peek(6).low_u64() as usize)), - instructions::DELEGATECALL => Some((stack.peek(4).low_u64() as usize, stack.peek(5).low_u64() as usize)), + let read = |pos| stack.peek(pos).low_u64() as usize; + let written = match instruction { + instructions::MSTORE | instructions::MLOAD => Some((read(0), 32)), + instructions::MSTORE8 => Some((read(0), 1)), + instructions::CALLDATACOPY | instructions::CODECOPY | instructions::RETURNDATACOPY => Some((read(0), read(2))), + instructions::EXTCODECOPY => Some((read(1), read(3))), + instructions::CALL | instructions::CALLCODE => Some((read(5), read(6))), + instructions::DELEGATECALL | instructions::STATICCALL => Some((read(4), read(5))), _ => None, + }; + + match written { + Some((offset, size)) if !memory::is_valid_range(offset, size) => None, + written => written, } } @@ -862,3 +873,36 @@ fn address_to_u256(value: Address) -> U256 { U256::from(&*H256::from(value)) } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use rustc_hex::FromHex; + use vmtype::VMType; + use factory::Factory; + use vm::{self, ActionParams, ActionValue}; + use vm::tests::{FakeExt, test_finalize}; + + #[test] + fn should_not_fail_on_tracing_mem() { + let code = "7feeffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff006000527faaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffaa6020526000620f120660406000601773945304eb96065b2a98b57a48a06ae28d285a71b56101f4f1600055".from_hex().unwrap(); + + let mut params = ActionParams::default(); + params.address = 5.into(); + params.gas = 300_000.into(); + params.gas_price = 1.into(); + params.value = ActionValue::Transfer(100_000.into()); + params.code = Some(Arc::new(code)); + let mut ext = FakeExt::new(); + ext.balances.insert(5.into(), 1_000_000_000.into()); + ext.tracing = true; + + let gas_left = { + let mut vm = Factory::new(VMType::Interpreter, 1).create(params.gas); + test_finalize(vm.exec(params, &mut ext)).unwrap() + }; + + assert_eq!(ext.calls.len(), 1); + assert_eq!(gas_left, 248_212.into()); + } +} diff --git a/ethcore/native_contracts/build.rs b/ethcore/native_contracts/build.rs index bcb64067c..e7985b388 100644 --- a/ethcore/native_contracts/build.rs +++ b/ethcore/native_contracts/build.rs @@ -28,6 +28,7 @@ const SERVICE_TRANSACTION_ABI: &'static str = include_str!("res/service_transact const SECRETSTORE_ACL_STORAGE_ABI: &'static str = include_str!("res/secretstore_acl_storage.json"); const VALIDATOR_SET_ABI: &'static str = include_str!("res/validator_set.json"); const VALIDATOR_REPORT_ABI: &'static str = include_str!("res/validator_report.json"); +const PEER_SET_ABI: &'static str = include_str!("res/peer_set.json"); const TEST_VALIDATOR_SET_ABI: &'static str = include_str!("res/test_validator_set.json"); @@ -53,6 +54,7 @@ fn main() { build_file("SecretStoreAclStorage", SECRETSTORE_ACL_STORAGE_ABI, "secretstore_acl_storage.rs"); build_file("ValidatorSet", VALIDATOR_SET_ABI, "validator_set.rs"); build_file("ValidatorReport", VALIDATOR_REPORT_ABI, "validator_report.rs"); + build_file("PeerSet", PEER_SET_ABI, "peer_set.rs"); build_test_contracts(); } diff --git a/ethcore/native_contracts/res/peer_set.json b/ethcore/native_contracts/res/peer_set.json new file mode 100644 index 000000000..a932f948d --- /dev/null +++ b/ethcore/native_contracts/res/peer_set.json @@ -0,0 +1 @@ +[{"constant":true,"inputs":[{"name":"sl","type":"bytes32"},{"name":"sh","type":"bytes32"},{"name":"pl","type":"bytes32"},{"name":"ph","type":"bytes32"}],"name":"connectionAllowed","outputs":[{"name":"res","type":"bool"}],"payable":false,"type":"function"},{"inputs":[],"payable":false,"type":"constructor"}] diff --git a/ethcore/native_contracts/src/lib.rs b/ethcore/native_contracts/src/lib.rs index 58875f8a2..733dea80e 100644 --- a/ethcore/native_contracts/src/lib.rs +++ b/ethcore/native_contracts/src/lib.rs @@ -30,6 +30,7 @@ mod service_transaction; mod secretstore_acl_storage; mod validator_set; mod validator_report; +mod peer_set; pub mod test_contracts; @@ -40,3 +41,4 @@ pub use self::service_transaction::ServiceTransactionChecker; pub use self::secretstore_acl_storage::SecretStoreAclStorage; pub use self::validator_set::ValidatorSet; pub use self::validator_report::ValidatorReport; +pub use self::peer_set::PeerSet; diff --git a/ethcore/native_contracts/src/peer_set.rs b/ethcore/native_contracts/src/peer_set.rs new file mode 100644 index 000000000..09d0ecbb8 --- /dev/null +++ b/ethcore/native_contracts/src/peer_set.rs @@ -0,0 +1,21 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![allow(unused_mut, unused_variables, unused_imports)] + +//! Peer set contract. + +include!(concat!(env!("OUT_DIR"), "/peer_set.rs")); diff --git a/ethcore/node_filter/Cargo.toml b/ethcore/node_filter/Cargo.toml new file mode 100644 index 000000000..e885ef1d1 --- /dev/null +++ b/ethcore/node_filter/Cargo.toml @@ -0,0 +1,16 @@ +[package] +description = "Parity smart network connections" +homepage = "http://parity.io" +license = "GPL-3.0" +name = "node-filter" +version = "1.8.0" +authors = ["Parity Technologies "] + +[dependencies] +ethcore = { path = ".."} +ethcore-util = { path = "../../util" } +ethcore-io = { path = "../../util/io" } +ethcore-network = { path = "../../util/network" } +native-contracts = { path = "../native_contracts" } +futures = "0.1" +log = "0.3" diff --git a/ethcore/node_filter/res/node_filter.json b/ethcore/node_filter/res/node_filter.json new file mode 100644 index 000000000..f8eb17152 --- /dev/null +++ b/ethcore/node_filter/res/node_filter.json @@ -0,0 +1,44 @@ +{ + "name": "TestNodeFilterContract", + "engine": { + "authorityRound": { + "params": { + "stepDuration": 1, + "startStep": 2, + "validators": { + "contract": "0x0000000000000000000000000000000000000005" + } + } + } + }, + "params": { + "accountStartNonce": "0x0", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID" : "0x69", + "gasLimitBoundDivisor": "0x0400" + }, + "genesis": { + "seal": { + "generic": "0xc180" + }, + "difficulty": "0x20000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x", + "gasLimit": "0x222222" + }, + "accounts": { + "0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "0000000000000000000000000000000000000005": { + "balance": "1", + "constructor": "6060604052341561000f57600080fd5b5b6012600102600080601160010260001916815260200190815260200160002081600019169055506022600102600080602160010260001916815260200190815260200160002081600019169055506032600102600080603160010260001916815260200190815260200160002081600019169055506042600102600080604160010260001916815260200190815260200160002081600019169055505b5b610155806100bd6000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063994d790a1461003e575b600080fd5b341561004957600080fd5b61008a6004808035600019169060200190919080356000191690602001909190803560001916906020019091908035600019169060200190919050506100a4565b604051808215151515815260200191505060405180910390f35b60006001800285600019161480156100c3575060026001028460001916145b156100d15760019050610121565b60006001028360001916141580156100f157506000600102826000191614155b801561011e5750816000191660008085600019166000191681526020019081526020016000205460001916145b90505b9493505050505600a165627a7a723058202082b8d8667fd397925f39785d8e804540beda0524d28af15921375145dfcc250029" + }, + "0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e": { "balance": "1606938044258990275541962092341162602522202993782792835301376" }, + "0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1": { "balance": "1606938044258990275541962092341162602522202993782792835301376" } + } +} diff --git a/ethcore/node_filter/src/lib.rs b/ethcore/node_filter/src/lib.rs new file mode 100644 index 000000000..d3dcbaa3b --- /dev/null +++ b/ethcore/node_filter/src/lib.rs @@ -0,0 +1,154 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Smart contract based node filter. + +extern crate ethcore; +extern crate ethcore_util as util; +extern crate ethcore_network as network; +extern crate native_contracts; +extern crate futures; +#[cfg(test)] extern crate ethcore_io as io; +#[macro_use] extern crate log; + +use std::sync::Weak; +use std::collections::HashMap; +use native_contracts::PeerSet as Contract; +use network::{NodeId, ConnectionFilter, ConnectionDirection}; +use ethcore::client::{BlockChainClient, BlockId, ChainNotify}; +use util::{Mutex, Address, H256, Bytes}; +use futures::Future; + +const MAX_CACHE_SIZE: usize = 4096; + +/// Connection filter that uses a contract to manage permissions. +pub struct NodeFilter { + contract: Mutex>, + client: Weak, + contract_address: Address, + permission_cache: Mutex>, +} + +impl NodeFilter { + /// Create a new instance. Accepts a contract address. + pub fn new(client: Weak, contract_address: Address) -> NodeFilter { + NodeFilter { + contract: Mutex::new(None), + client: client, + contract_address: contract_address, + permission_cache: Mutex::new(HashMap::new()), + } + } + + /// Clear cached permissions. + pub fn clear_cache(&self) { + self.permission_cache.lock().clear(); + } +} + +impl ConnectionFilter for NodeFilter { + fn connection_allowed(&self, own_id: &NodeId, connecting_id: &NodeId, _direction: ConnectionDirection) -> bool { + + let mut cache = self.permission_cache.lock(); + if let Some(res) = cache.get(connecting_id) { + return *res; + } + + let mut contract = self.contract.lock(); + if contract.is_none() { + *contract = Some(Contract::new(self.contract_address)); + } + + let allowed = match (self.client.upgrade(), &*contract) { + (Some(ref client), &Some(ref contract)) => { + let own_low = H256::from_slice(&own_id[0..32]); + let own_high = H256::from_slice(&own_id[32..64]); + let id_low = H256::from_slice(&connecting_id[0..32]); + let id_high = H256::from_slice(&connecting_id[32..64]); + let allowed = contract.connection_allowed( + |addr, data| futures::done(client.call_contract(BlockId::Latest, addr, data)), + own_low, + own_high, + id_low, + id_high, + ).wait().unwrap_or_else(|e| { + debug!("Error callling peer set contract: {:?}", e); + false + }); + + allowed + } + _ => false, + }; + + if cache.len() < MAX_CACHE_SIZE { + cache.insert(*connecting_id, allowed); + } + allowed + } +} + +impl ChainNotify for NodeFilter { + fn new_blocks(&self, imported: Vec, _invalid: Vec, _enacted: Vec, _retracted: Vec, _sealed: Vec, _proposed: Vec, _duration: u64) { + if !imported.is_empty() { + self.clear_cache(); + } + } +} + + +#[cfg(test)] +mod test { + use std::sync::{Arc, Weak}; + use std::str::FromStr; + use ethcore::spec::Spec; + use ethcore::client::{BlockChainClient, Client, ClientConfig}; + use ethcore::miner::Miner; + use util::{Address}; + use network::{ConnectionDirection, ConnectionFilter, NodeId}; + use io::IoChannel; + use super::NodeFilter; + + /// Contract code: https://gist.github.com/arkpar/467dbcc73cbb85b0997a7a10ffa0695f + #[test] + fn node_filter() { + let contract_addr = Address::from_str("0000000000000000000000000000000000000005").unwrap(); + let data = include_bytes!("../res/node_filter.json"); + let spec = Spec::load(::std::env::temp_dir(), &data[..]).unwrap(); + let client_db = Arc::new(::util::kvdb::in_memory(::ethcore::db::NUM_COLUMNS.unwrap_or(0))); + + let client = Client::new( + ClientConfig::default(), + &spec, + client_db, + Arc::new(Miner::with_spec(&spec)), + IoChannel::disconnected(), + ).unwrap(); + let filter = NodeFilter::new(Arc::downgrade(&client) as Weak, contract_addr); + let self1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002").unwrap(); + let self2 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003").unwrap(); + let node1 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012").unwrap(); + let node2 = NodeId::from_str("00000000000000000000000000000000000000000000000000000000000000210000000000000000000000000000000000000000000000000000000000000022").unwrap(); + let nodex = NodeId::from_str("77000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); + + assert!(filter.connection_allowed(&self1, &node1, ConnectionDirection::Inbound)); + assert!(filter.connection_allowed(&self1, &nodex, ConnectionDirection::Inbound)); + filter.clear_cache(); + assert!(filter.connection_allowed(&self2, &node1, ConnectionDirection::Inbound)); + assert!(filter.connection_allowed(&self2, &node2, ConnectionDirection::Inbound)); + assert!(!filter.connection_allowed(&self2, &nodex, ConnectionDirection::Inbound)); + } +} diff --git a/ethcore/res/wasm-tests b/ethcore/res/wasm-tests index 85e76c5ea..519b0b967 160000 --- a/ethcore/res/wasm-tests +++ b/ethcore/res/wasm-tests @@ -1 +1 @@ -Subproject commit 85e76c5ea2a54c6c54e35014643b5080a50460c5 +Subproject commit 519b0b967cffd7d1236ef21698b1e6e415a048e9 diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 2355fbfdf..c3a70748c 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -748,7 +748,7 @@ impl Client { self.factories.clone(), ).expect("state known to be available for just-imported block; qed"); - let options = TransactOptions { tracing: false, vm_tracing: false, check_nonce: false }; + let options = TransactOptions::with_no_tracing().dont_check_nonce(); let res = Executive::new(&mut state, &env_info, &*self.engine) .transact(&transaction, options); @@ -1113,18 +1113,39 @@ impl Client { }.fake_sign(from) } - fn do_call(&self, env_info: &EnvInfo, state: &mut State, t: &SignedTransaction, analytics: CallAnalytics) -> Result { - let original_state = if analytics.state_diffing { Some(state.clone()) } else { None }; + fn do_virtual_call(&self, env_info: &EnvInfo, state: &mut State, t: &SignedTransaction, analytics: CallAnalytics) -> Result { + fn call( + state: &mut State, + env_info: &EnvInfo, + engine: &E, + state_diff: bool, + transaction: &SignedTransaction, + options: TransactOptions, + ) -> Result where + E: Engine + ?Sized, + T: trace::Tracer, + V: trace::VMTracer, + { + let options = options.dont_check_nonce(); + let original_state = if state_diff { Some(state.clone()) } else { None }; - let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false }; - let mut ret = Executive::new(state, env_info, &*self.engine).transact_virtual(t, options)?; + let mut ret = Executive::new(state, env_info, engine).transact_virtual(transaction, options)?; - // TODO gav move this into Executive. - if let Some(original) = original_state { - ret.state_diff = Some(state.diff_from(original).map_err(ExecutionError::from)?); + if let Some(original) = original_state { + ret.state_diff = Some(state.diff_from(original).map_err(ExecutionError::from)?); + } + Ok(ret) } - Ok(ret) + let state_diff = analytics.state_diffing; + let engine = &*self.engine; + + match (analytics.transaction_tracing, analytics.vm_tracing) { + (true, true) => call(state, env_info, engine, state_diff, t, TransactOptions::with_tracing_and_vm_tracing()), + (true, false) => call(state, env_info, engine, state_diff, t, TransactOptions::with_tracing()), + (false, true) => call(state, env_info, engine, state_diff, t, TransactOptions::with_vm_tracing()), + (false, false) => call(state, env_info, engine, state_diff, t, TransactOptions::with_no_tracing()), + } } } @@ -1157,7 +1178,7 @@ impl BlockChainClient for Client { // that's just a copy of the state. let mut state = self.state_at(block).ok_or(CallError::StatePruned)?; - self.do_call(&env_info, &mut state, transaction, analytics) + self.do_virtual_call(&env_info, &mut state, transaction, analytics) } fn call_many(&self, transactions: &[(SignedTransaction, CallAnalytics)], block: BlockId) -> Result, CallError> { @@ -1169,7 +1190,7 @@ impl BlockChainClient for Client { let mut results = Vec::with_capacity(transactions.len()); for &(ref t, analytics) in transactions { - let ret = self.do_call(&env_info, &mut state, t, analytics)?; + let ret = self.do_virtual_call(&env_info, &mut state, t, analytics)?; env_info.gas_used = ret.cumulative_gas_used; results.push(ret); } @@ -1189,7 +1210,7 @@ impl BlockChainClient for Client { // that's just a copy of the state. let original_state = self.state_at(block).ok_or(CallError::StatePruned)?; let sender = t.sender(); - let options = TransactOptions { tracing: true, vm_tracing: false, check_nonce: false }; + let options = || TransactOptions::with_tracing(); let cond = |gas| { let mut tx = t.as_unsigned().clone(); @@ -1198,7 +1219,7 @@ impl BlockChainClient for Client { let mut state = original_state.clone(); Ok(Executive::new(&mut state, &env_info, &*self.engine) - .transact_virtual(&tx, options.clone()) + .transact_virtual(&tx, options()) .map(|r| r.exception.is_none()) .unwrap_or(false)) }; @@ -1254,22 +1275,17 @@ impl BlockChainClient for Client { return Err(CallError::TransactionNotFound); } - let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false }; const PROOF: &'static str = "Transactions fetched from blockchain; blockchain transactions are valid; qed"; let rest = txs.split_off(address.index); for t in txs { let t = SignedTransaction::new(t).expect(PROOF); - let x = Executive::new(&mut state, &env_info, &*self.engine).transact(&t, Default::default())?; + let x = Executive::new(&mut state, &env_info, &*self.engine).transact(&t, TransactOptions::with_no_tracing())?; env_info.gas_used = env_info.gas_used + x.gas_used; } let first = rest.into_iter().next().expect("We split off < `address.index`; Length is checked earlier; qed"); let t = SignedTransaction::new(first).expect(PROOF); - let original_state = if analytics.state_diffing { Some(state.clone()) } else { None }; - let mut ret = Executive::new(&mut state, &env_info, &*self.engine).transact(&t, options)?; - if let Some(original) = original_state { - ret.state_diff = Some(state.diff_from(original).map_err(ExecutionError::from)?) - } - Ok(ret) + + self.do_virtual_call(&env_info, &mut state, &t, analytics) } fn mode(&self) -> IpcMode { diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index cd8501c31..a455a3724 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -18,9 +18,9 @@ use std::fmt; use std::sync::Arc; -use util::{self, U256, journaldb, trie}; +use util::{self, U256, H256, journaldb, trie}; use util::kvdb::{self, KeyValueDB}; -use {state, state_db, client, executive, trace, db, spec}; +use {state, state_db, client, executive, trace, transaction, db, spec, pod_state}; use factory::Factories; use evm::{self, VMType}; use vm::{self, ActionParams}; @@ -33,9 +33,17 @@ pub enum EvmTestError { /// EVM error. Evm(vm::Error), /// Initialization error. - Initialization(::error::Error), + ClientError(::error::Error), /// Low-level database error. Database(String), + /// Post-condition failure, + PostCondition(String), +} + +impl> From for EvmTestError { + fn from(err: E) -> Self { + EvmTestError::ClientError(err.into()) + } } impl fmt::Display for EvmTestError { @@ -45,52 +53,114 @@ impl fmt::Display for EvmTestError { match *self { Trie(ref err) => write!(fmt, "Trie: {}", err), Evm(ref err) => write!(fmt, "EVM: {}", err), - Initialization(ref err) => write!(fmt, "Initialization: {}", err), + ClientError(ref err) => write!(fmt, "{}", err), Database(ref err) => write!(fmt, "DB: {}", err), + PostCondition(ref err) => write!(fmt, "{}", err), } } } -/// Simplified, single-block EVM test client. -pub struct EvmTestClient { - state_db: state_db::StateDB, - factories: Factories, - spec: spec::Spec, +use ethereum; +use ethjson::state::test::ForkSpec; + +lazy_static! { + pub static ref FRONTIER: spec::Spec = ethereum::new_frontier_test(); + pub static ref HOMESTEAD: spec::Spec = ethereum::new_homestead_test(); + pub static ref EIP150: spec::Spec = ethereum::new_eip150_test(); + pub static ref EIP161: spec::Spec = ethereum::new_eip161_test(); + pub static ref _METROPOLIS: spec::Spec = ethereum::new_metropolis_test(); } -impl EvmTestClient { - /// Creates new EVM test client with in-memory DB initialized with genesis of given Spec. - pub fn new(spec: spec::Spec) -> Result { - let factories = Factories { - vm: evm::Factory::new(VMType::Interpreter, 5 * 1024), - trie: trie::TrieFactory::new(trie::TrieSpec::Secure), - accountdb: Default::default(), - }; - let db = Arc::new(kvdb::in_memory(db::NUM_COLUMNS.expect("We use column-based DB; qed"))); - let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, db::COL_STATE); - let mut state_db = state_db::StateDB::new(journal_db, 5 * 1024 * 1024); - state_db = spec.ensure_db_good(state_db, &factories).map_err(EvmTestError::Initialization)?; - // Write DB - { - let mut batch = kvdb::DBTransaction::new(); - state_db.journal_under(&mut batch, 0, &spec.genesis_header().hash()).map_err(|e| EvmTestError::Initialization(e.into()))?; - db.write(batch).map_err(EvmTestError::Database)?; +/// Simplified, single-block EVM test client. +pub struct EvmTestClient<'a> { + state: state::State, + spec: &'a spec::Spec, +} + +impl<'a> EvmTestClient<'a> { + /// Converts a json spec definition into spec. + pub fn spec_from_json(spec: &ForkSpec) -> Option<&'static spec::Spec> { + match *spec { + ForkSpec::Frontier => Some(&*FRONTIER), + ForkSpec::Homestead => Some(&*HOMESTEAD), + ForkSpec::EIP150 => Some(&*EIP150), + ForkSpec::EIP158 => Some(&*EIP161), + ForkSpec::Metropolis | ForkSpec::Byzantium | ForkSpec::Constantinople => None, } + } + + /// Creates new EVM test client with in-memory DB initialized with genesis of given Spec. + pub fn new(spec: &'a spec::Spec) -> Result { + let factories = Self::factories(); + let state = Self::state_from_spec(spec, &factories)?; Ok(EvmTestClient { - state_db, - factories, + state, spec, }) } - /// Call given contract. + /// Creates new EVM test client with in-memory DB initialized with given PodState. + pub fn from_pod_state(spec: &'a spec::Spec, pod_state: pod_state::PodState) -> Result { + let factories = Self::factories(); + let state = Self::state_from_pod(spec, &factories, pod_state)?; + + Ok(EvmTestClient { + state, + spec, + }) + } + + fn factories() -> Factories { + Factories { + vm: evm::Factory::new(VMType::Interpreter, 5 * 1024), + trie: trie::TrieFactory::new(trie::TrieSpec::Secure), + accountdb: Default::default(), + } + } + + fn state_from_spec(spec: &'a spec::Spec, factories: &Factories) -> Result, EvmTestError> { + let db = Arc::new(kvdb::in_memory(db::NUM_COLUMNS.expect("We use column-based DB; qed"))); + let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, db::COL_STATE); + let mut state_db = state_db::StateDB::new(journal_db, 5 * 1024 * 1024); + state_db = spec.ensure_db_good(state_db, factories)?; + + let genesis = spec.genesis_header(); + // Write DB + { + let mut batch = kvdb::DBTransaction::new(); + state_db.journal_under(&mut batch, 0, &genesis.hash())?; + db.write(batch).map_err(EvmTestError::Database)?; + } + + state::State::from_existing( + state_db, + *genesis.state_root(), + spec.engine.account_start_nonce(0), + factories.clone() + ).map_err(EvmTestError::Trie) + } + + fn state_from_pod(spec: &'a spec::Spec, factories: &Factories, pod_state: pod_state::PodState) -> Result, EvmTestError> { + let db = Arc::new(kvdb::in_memory(db::NUM_COLUMNS.expect("We use column-based DB; qed"))); + let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, db::COL_STATE); + let state_db = state_db::StateDB::new(journal_db, 5 * 1024 * 1024); + let mut state = state::State::new( + state_db, + spec.engine.account_start_nonce(0), + factories.clone(), + ); + state.populate_from(pod_state); + state.commit()?; + Ok(state) + } + + /// Execute the VM given ActionParams and tracer. + /// Returns amount of gas left and the output. pub fn call(&mut self, params: ActionParams, vm_tracer: &mut T) -> Result<(U256, Vec), EvmTestError> { let genesis = self.spec.genesis_header(); - let mut state = state::State::from_existing(self.state_db.boxed_clone(), *genesis.state_root(), self.spec.engine.account_start_nonce(0), self.factories.clone()) - .map_err(EvmTestError::Trie)?; let info = client::EnvInfo { number: genesis.number(), author: *genesis.author(), @@ -103,7 +173,7 @@ impl EvmTestClient { let mut substate = state::Substate::new(); let mut tracer = trace::NoopTracer; let mut output = vec![]; - let mut executive = executive::Executive::new(&mut state, &info, &*self.spec.engine); + let mut executive = executive::Executive::new(&mut self.state, &info, &*self.spec.engine); let (gas_left, _) = executive.call( params, &mut substate, @@ -114,4 +184,59 @@ impl EvmTestClient { Ok((gas_left, output)) } + + /// Executes a SignedTransaction within context of the provided state and `EnvInfo`. + /// Returns the state root, gas left and the output. + pub fn transact( + &mut self, + env_info: &client::EnvInfo, + transaction: transaction::SignedTransaction, + vm_tracer: T, + ) -> TransactResult { + let initial_gas = transaction.gas; + // Verify transaction + let is_ok = transaction.verify_basic(true, None, env_info.number >= self.spec.engine.params().eip86_transition); + if let Err(error) = is_ok { + return TransactResult::Err { + state_root: *self.state.root(), + error, + }; + } + + // Apply transaction + let tracer = trace::NoopTracer; + let result = self.state.apply_with_tracing(&env_info, &*self.spec.engine, &transaction, tracer, vm_tracer); + + match result { + Ok(result) => TransactResult::Ok { + state_root: *self.state.root(), + gas_left: initial_gas - result.receipt.gas_used, + output: result.output + }, + Err(error) => TransactResult::Err { + state_root: *self.state.root(), + error, + }, + } + } +} + +/// A result of applying transaction to the state. +pub enum TransactResult { + /// Successful execution + Ok { + /// State root + state_root: H256, + /// Amount of gas left + gas_left: U256, + /// Output + output: Vec, + }, + /// Transaction failed to run + Err { + /// State root + state_root: H256, + /// Execution error + error: ::error::Error, + }, } diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index f7c7417ad..9377c2f44 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -27,7 +27,7 @@ mod client; pub use self::client::*; pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType}; pub use self::error::Error; -pub use self::evm_test_client::{EvmTestClient, EvmTestError}; +pub use self::evm_test_client::{EvmTestClient, EvmTestError, TransactResult}; pub use self::test_client::{TestBlockChainClient, EachBlockWith}; pub use self::chain_notify::ChainNotify; pub use self::traits::{BlockChainClient, MiningBlockChainClient, EngineClient}; diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 6235255e4..fc7b27403 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -187,7 +187,14 @@ impl Engine for Arc { /// Additional engine-specific information for the user/developer concerning `header`. fn extra_info(&self, header: &Header) -> BTreeMap { - map!["nonce".to_owned() => format!("0x{}", header.nonce().hex()), "mixHash".to_owned() => format!("0x{}", header.mix_hash().hex())] + if header.seal().len() == self.seal_fields() { + map![ + "nonce".to_owned() => format!("0x{}", header.nonce().hex()), + "mixHash".to_owned() => format!("0x{}", header.mix_hash().hex()) + ] + } else { + BTreeMap::default() + } } fn schedule(&self, block_number: BlockNumber) -> Schedule { diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index 8f2fb06f2..66a770243 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -26,7 +26,7 @@ use evm::{CallType, Factory, Finalize, FinalizationResult}; use vm::{self, Ext, CreateContractAddress, ReturnData, CleanDustMode, ActionParams, ActionValue}; use wasm; use externalities::*; -use trace::{FlatTrace, Tracer, NoopTracer, ExecutiveTracer, VMTrace, VMTracer, ExecutiveVMTracer, NoopVMTracer}; +use trace::{self, FlatTrace, VMTrace, Tracer, VMTracer}; use transaction::{Action, SignedTransaction}; use crossbeam; pub use executed::{Executed, ExecutionResult}; @@ -66,16 +66,77 @@ pub fn contract_address(address_scheme: CreateContractAddress, sender: &Address, } /// Transaction execution options. -#[derive(Default, Copy, Clone, PartialEq)] -pub struct TransactOptions { +#[derive(Copy, Clone, PartialEq)] +pub struct TransactOptions { /// Enable call tracing. - pub tracing: bool, + pub tracer: T, /// Enable VM tracing. - pub vm_tracing: bool, + pub vm_tracer: V, /// Check transaction nonce before execution. pub check_nonce: bool, } +impl TransactOptions { + /// Create new `TransactOptions` with given tracer and VM tracer. + pub fn new(tracer: T, vm_tracer: V) -> Self { + TransactOptions { + tracer, + vm_tracer, + check_nonce: true, + } + } + + /// Disables the nonce check + pub fn dont_check_nonce(mut self) -> Self { + self.check_nonce = false; + self + } +} + +impl TransactOptions { + /// Creates new `TransactOptions` with default tracing and VM tracing. + pub fn with_tracing_and_vm_tracing() -> Self { + TransactOptions { + tracer: trace::ExecutiveTracer::default(), + vm_tracer: trace::ExecutiveVMTracer::toplevel(), + check_nonce: true, + } + } +} + +impl TransactOptions { + /// Creates new `TransactOptions` with default tracing and no VM tracing. + pub fn with_tracing() -> Self { + TransactOptions { + tracer: trace::ExecutiveTracer::default(), + vm_tracer: trace::NoopVMTracer, + check_nonce: true, + } + } +} + +impl TransactOptions { + /// Creates new `TransactOptions` with no tracing and default VM tracing. + pub fn with_vm_tracing() -> Self { + TransactOptions { + tracer: trace::NoopTracer, + vm_tracer: trace::ExecutiveVMTracer::toplevel(), + check_nonce: true, + } + } +} + +impl TransactOptions { + /// Creates new `TransactOptions` without any tracing. + pub fn with_no_tracing() -> Self { + TransactOptions { + tracer: trace::NoopTracer, + vm_tracer: trace::NoopVMTracer, + check_nonce: true, + } + } +} + pub fn executor(engine: &E, vm_factory: &Factory, params: &ActionParams) -> Box where E: Engine + ?Sized { @@ -137,24 +198,18 @@ impl<'a, B: 'a + StateBackend, E: Engine + ?Sized> Executive<'a, B, E> { } /// This function should be used to execute transaction. - pub fn transact(&'a mut self, t: &SignedTransaction, options: TransactOptions) -> Result { - let check = options.check_nonce; - match options.tracing { - true => match options.vm_tracing { - true => self.transact_with_tracer(t, check, ExecutiveTracer::default(), ExecutiveVMTracer::toplevel()), - false => self.transact_with_tracer(t, check, ExecutiveTracer::default(), NoopVMTracer), - }, - false => match options.vm_tracing { - true => self.transact_with_tracer(t, check, NoopTracer, ExecutiveVMTracer::toplevel()), - false => self.transact_with_tracer(t, check, NoopTracer, NoopVMTracer), - }, - } + pub fn transact(&'a mut self, t: &SignedTransaction, options: TransactOptions) + -> Result where T: Tracer, V: VMTracer, + { + self.transact_with_tracer(t, options.check_nonce, options.tracer, options.vm_tracer) } /// Execute a transaction in a "virtual" context. /// This will ensure the caller has enough balance to execute the desired transaction. /// Used for extra-block executions for things like consensus contracts and RPCs - pub fn transact_virtual(&'a mut self, t: &SignedTransaction, options: TransactOptions) -> Result { + pub fn transact_virtual(&'a mut self, t: &SignedTransaction, options: TransactOptions) + -> Result where T: Tracer, V: VMTracer, + { let sender = t.sender(); let balance = self.state.balance(&sender)?; let needed_balance = t.value.saturating_add(t.gas.saturating_mul(t.gas_price)); @@ -167,7 +222,7 @@ impl<'a, B: 'a + StateBackend, E: Engine + ?Sized> Executive<'a, B, E> { } /// Execute transaction/call with tracing enabled - pub fn transact_with_tracer( + fn transact_with_tracer( &'a mut self, t: &SignedTransaction, check_nonce: bool, @@ -261,7 +316,7 @@ impl<'a, B: 'a + StateBackend, E: Engine + ?Sized> Executive<'a, B, E> { }; // finalize here! - Ok(self.finalize(t, substate, result, output, tracer.traces(), vm_tracer.drain())?) + Ok(self.finalize(t, substate, result, output, tracer.drain(), vm_tracer.drain())?) } fn exec_vm( @@ -399,7 +454,7 @@ impl<'a, B: 'a + StateBackend, E: Engine + ?Sized> Executive<'a, B, E> { trace!(target: "executive", "res={:?}", res); - let traces = subtracer.traces(); + let traces = subtracer.drain(); match res { Ok(ref res) => tracer.trace_call( trace_info, @@ -484,9 +539,9 @@ impl<'a, B: 'a + StateBackend, E: Engine + ?Sized> Executive<'a, B, E> { gas - res.gas_left, trace_output, created, - subtracer.traces() + subtracer.drain() ), - Err(ref e) => tracer.trace_failed_create(trace_info, subtracer.traces(), e.into()) + Err(ref e) => tracer.trace_failed_create(trace_info, subtracer.drain(), e.into()) }; self.enact_result(&res, substate, unconfirmed_substate); @@ -794,7 +849,7 @@ mod tests { }), }]; - assert_eq!(tracer.traces(), expected_trace); + assert_eq!(tracer.drain(), expected_trace); let expected_vm_trace = VMTrace { parent_step: 0, @@ -887,7 +942,7 @@ mod tests { }), }]; - assert_eq!(tracer.traces(), expected_trace); + assert_eq!(tracer.drain(), expected_trace); let expected_vm_trace = VMTrace { parent_step: 0, @@ -1138,7 +1193,7 @@ mod tests { let executed = { let mut ex = Executive::new(&mut state, &info, &engine); - let opts = TransactOptions { check_nonce: true, tracing: false, vm_tracing: false }; + let opts = TransactOptions::with_no_tracing(); ex.transact(&t, opts).unwrap() }; @@ -1175,7 +1230,7 @@ mod tests { let res = { let mut ex = Executive::new(&mut state, &info, &engine); - let opts = TransactOptions { check_nonce: true, tracing: false, vm_tracing: false }; + let opts = TransactOptions::with_no_tracing(); ex.transact(&t, opts) }; @@ -1208,7 +1263,7 @@ mod tests { let res = { let mut ex = Executive::new(&mut state, &info, &engine); - let opts = TransactOptions { check_nonce: true, tracing: false, vm_tracing: false }; + let opts = TransactOptions::with_no_tracing(); ex.transact(&t, opts) }; @@ -1241,7 +1296,7 @@ mod tests { let res = { let mut ex = Executive::new(&mut state, &info, &engine); - let opts = TransactOptions { check_nonce: true, tracing: false, vm_tracing: false }; + let opts = TransactOptions::with_no_tracing(); ex.transact(&t, opts) }; diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 2fdf8875f..51961a2bb 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -15,23 +15,13 @@ // along with Parity. If not, see . use super::test_common::*; -use tests::helpers::*; use pod_state::PodState; -use ethereum; -use spec::Spec; +use trace; +use client::{EvmTestClient, EvmTestError, TransactResult}; use ethjson; -use ethjson::state::test::ForkSpec; use transaction::SignedTransaction; use vm::EnvInfo; -lazy_static! { - pub static ref FRONTIER: Spec = ethereum::new_frontier_test(); - pub static ref HOMESTEAD: Spec = ethereum::new_homestead_test(); - pub static ref EIP150: Spec = ethereum::new_eip150_test(); - pub static ref EIP161: Spec = ethereum::new_eip161_test(); - pub static ref _METROPOLIS: Spec = ethereum::new_metropolis_test(); -} - pub fn json_chain_test(json_data: &[u8]) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); @@ -43,35 +33,49 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { let env: EnvInfo = test.env.into(); let pre: PodState = test.pre_state.into(); - for (spec, states) in test.post_states { + for (spec_name, states) in test.post_states { let total = states.len(); - let engine = match spec { - ForkSpec::Frontier => &FRONTIER.engine, - ForkSpec::Homestead => &HOMESTEAD.engine, - ForkSpec::EIP150 => &EIP150.engine, - ForkSpec::EIP158 => &EIP161.engine, - ForkSpec::Metropolis => continue, + let spec = match EvmTestClient::spec_from_json(&spec_name) { + Some(spec) => spec, + None => { + println!(" - {} | {:?} Ignoring tests because of missing spec", name, spec_name); + continue; + } }; for (i, state) in states.into_iter().enumerate() { - let info = format!(" - {} | {:?} ({}/{}) ...", name, spec, i + 1, total); + let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); let post_root: H256 = state.hash.into(); let transaction: SignedTransaction = multitransaction.select(&state.indexes).into(); - let mut state = get_temp_state(); - state.populate_from(pre.clone()); - if transaction.verify_basic(true, None, env.number >= engine.params().eip86_transition).is_ok() { - state.commit().expect(&format!("State test {} failed due to internal error.", name)); - let _res = state.apply(&env, &**engine, &transaction, false); - } else { - let _rest = state.commit(); - } - if state.root() != &post_root { - println!("{} !!! State mismatch (got: {}, expect: {}", info, state.root(), post_root); - flushln!("{} fail", info); - failed.push(name.clone()); - } else { - flushln!("{} ok", info); + + let result = || -> Result<_, EvmTestError> { + Ok(EvmTestClient::from_pod_state(spec, pre.clone())? + .transact(&env, transaction, trace::NoopVMTracer)) + }; + match result() { + Err(err) => { + println!("{} !!! Unexpected internal error: {:?}", info, err); + flushln!("{} fail", info); + failed.push(name.clone()); + }, + Ok(TransactResult::Ok { state_root, .. }) if state_root != post_root => { + println!("{} !!! State mismatch (got: {}, expect: {}", info, state_root, post_root); + flushln!("{} fail", info); + failed.push(name.clone()); + }, + Ok(TransactResult::Err { state_root, ref error }) if state_root != post_root => { + println!("{} !!! State mismatch (got: {}, expect: {}", info, state_root, post_root); + println!("{} !!! Execution error: {:?}", info, error); + flushln!("{} fail", info); + failed.push(name.clone()); + }, + Ok(TransactResult::Err { error, .. }) => { + flushln!("{} ok ({:?})", info, error); + }, + Ok(_) => { + flushln!("{} ok", info); + }, } } } diff --git a/ethcore/src/miner/transaction_queue.rs b/ethcore/src/miner/transaction_queue.rs index 263143ee8..00029e70f 100644 --- a/ethcore/src/miner/transaction_queue.rs +++ b/ethcore/src/miner/transaction_queue.rs @@ -506,8 +506,6 @@ pub struct AccountDetails { pub balance: U256, } -/// Transactions with `gas > (gas_limit + gas_limit * Factor(in percents))` are not imported to the queue. -const GAS_LIMIT_HYSTERESIS: usize = 200; // (100/GAS_LIMIT_HYSTERESIS) % /// Transaction with the same (sender, nonce) can be replaced only if /// `new_gas_price > old_gas_price + old_gas_price >> SHIFT` const GAS_PRICE_BUMP_SHIFT: usize = 3; // 2 = 25%, 3 = 12.5%, 4 = 6.25% @@ -570,8 +568,8 @@ pub struct TransactionQueue { minimal_gas_price: U256, /// The maximum amount of gas any individual transaction may use. tx_gas_limit: U256, - /// Current gas limit (block gas limit * factor). Transactions above the limit will not be accepted (default to !0) - total_gas_limit: U256, + /// Current gas limit (block gas limit). Transactions above the limit will not be accepted (default to !0) + block_gas_limit: U256, /// Maximal time transaction may occupy the queue. /// When we reach `max_time_in_queue / 2^3` we re-validate /// account balance. @@ -631,7 +629,7 @@ impl TransactionQueue { TransactionQueue { strategy, minimal_gas_price: U256::zero(), - total_gas_limit: !U256::zero(), + block_gas_limit: !U256::zero(), tx_gas_limit, max_time_in_queue: DEFAULT_QUEUING_PERIOD, current, @@ -674,16 +672,10 @@ impl TransactionQueue { self.current.gas_price_entry_limit() } - /// Sets new gas limit. Transactions with gas slightly (`GAS_LIMIT_HYSTERESIS`) above the limit won't be imported. + /// Sets new gas limit. Transactions with gas over the limit will not be accepted. /// Any transaction already imported to the queue is not affected. pub fn set_gas_limit(&mut self, gas_limit: U256) { - let extra = gas_limit / U256::from(GAS_LIMIT_HYSTERESIS); - - let total_gas_limit = match gas_limit.overflowing_add(extra) { - (_, true) => !U256::zero(), - (val, false) => val, - }; - self.total_gas_limit = total_gas_limit; + self.block_gas_limit = gas_limit; } /// Sets new total gas limit. @@ -819,13 +811,13 @@ impl TransactionQueue { })); } - let gas_limit = cmp::min(self.tx_gas_limit, self.total_gas_limit); + let gas_limit = cmp::min(self.tx_gas_limit, self.block_gas_limit); if tx.gas > gas_limit { trace!(target: "txqueue", "Dropping transaction above gas limit: {:?} ({} > min({}, {}))", tx.hash(), tx.gas, - self.total_gas_limit, + self.block_gas_limit, self.tx_gas_limit ); return Err(Error::Transaction(TransactionError::GasLimitExceeded { @@ -1922,13 +1914,13 @@ pub mod test { // given let mut txq = TransactionQueue::default(); txq.set_gas_limit(U256::zero()); - assert_eq!(txq.total_gas_limit, U256::zero()); + assert_eq!(txq.block_gas_limit, U256::zero()); // when txq.set_gas_limit(!U256::zero()); // then - assert_eq!(txq.total_gas_limit, !U256::zero()); + assert_eq!(txq.block_gas_limit, !U256::zero()); } #[test] @@ -1945,7 +1937,7 @@ pub mod test { // then assert_eq!(unwrap_tx_err(res), TransactionError::GasLimitExceeded { - limit: U256::from(50_250), // Should be 100.5% of set_gas_limit + limit: U256::from(50_000), got: gas, }); let stats = txq.status(); diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 7dcd0c6aa..9cec548a6 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -99,6 +99,8 @@ pub struct CommonParams { pub block_reward: U256, /// Registrar contract address. pub registrar: Address, + /// Node permission managing contract address. + pub node_permission_contract: Option
, } impl CommonParams { @@ -170,6 +172,7 @@ impl From for CommonParams { gas_limit_bound_divisor: p.gas_limit_bound_divisor.into(), block_reward: p.block_reward.map_or_else(U256::zero, Into::into), registrar: p.registrar.map_or_else(Address::new, Into::into), + node_permission_contract: p.node_permission_contract.map(Into::into), } } } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 3898572a6..bb7f87574 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -31,7 +31,7 @@ use vm::EnvInfo; use error::Error; use executive::{Executive, TransactOptions}; use factory::Factories; -use trace::FlatTrace; +use trace::{self, FlatTrace, VMTrace}; use pod_account::*; use pod_state::{self, PodState}; use types::basic_account::BasicAccount; @@ -59,8 +59,12 @@ pub use self::substate::Substate; pub struct ApplyOutcome { /// The receipt for the applied transaction. pub receipt: Receipt, - /// The trace for the applied transaction, if None if tracing is disabled. + /// The output of the applied transaction. + pub output: Bytes, + /// The trace for the applied transaction, empty if tracing was not produced. pub trace: Vec, + /// The VM trace for the applied transaction, None if tracing was not produced. + pub vm_trace: Option } /// Result type for the execution ("application") of a transaction. @@ -205,7 +209,7 @@ pub fn check_proof( Err(_) => return ProvedExecution::BadProof, }; - match state.execute(env_info, engine, transaction, false, true) { + match state.execute(env_info, engine, transaction, TransactOptions::with_no_tracing(), true) { Ok(executed) => ProvedExecution::Complete(executed), Err(ExecutionError::Internal(_)) => ProvedExecution::BadProof, Err(e) => ProvedExecution::Failed(e), @@ -239,7 +243,8 @@ pub fn prove_transaction( Err(_) => return None, }; - match state.execute(env_info, engine, transaction, false, virt) { + let options = TransactOptions::with_no_tracing().dont_check_nonce(); + match state.execute(env_info, engine, transaction, options, virt) { Err(ExecutionError::Internal(_)) => None, Err(e) => { trace!(target: "state", "Proved call failed: {}", e); @@ -327,7 +332,7 @@ const SEC_TRIE_DB_UNWRAP_STR: &'static str = "A state can only be created with v impl State { /// Creates new state with empty state root - #[cfg(test)] + /// Used for tests. pub fn new(mut db: B, account_start_nonce: U256, factories: Factories) -> State { let mut root = H256::new(); { @@ -660,29 +665,57 @@ impl State { /// Execute a given transaction, producing a receipt and an optional trace. /// This will change the state accordingly. pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool) -> ApplyResult { -// let old = self.to_pod(); + if tracing { + let options = TransactOptions::with_tracing(); + self.apply_with_tracing(env_info, engine, t, options.tracer, options.vm_tracer) + } else { + let options = TransactOptions::with_no_tracing(); + self.apply_with_tracing(env_info, engine, t, options.tracer, options.vm_tracer) + } + } + + /// Execute a given transaction with given tracer and VM tracer producing a receipt and an optional trace. + /// This will change the state accordingly. + pub fn apply_with_tracing( + &mut self, + env_info: &EnvInfo, + engine: &Engine, + t: &SignedTransaction, + tracer: T, + vm_tracer: V, + ) -> ApplyResult where + T: trace::Tracer, + V: trace::VMTracer, + { + let options = TransactOptions::new(tracer, vm_tracer); + let e = self.execute(env_info, engine, t, options, false)?; - let e = self.execute(env_info, engine, t, tracing, false)?; -// trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod())); let state_root = if env_info.number < engine.params().eip98_transition || env_info.number < engine.params().validate_receipts_transition { self.commit()?; Some(self.root().clone()) } else { None }; + + let output = e.output; let receipt = Receipt::new(state_root, e.cumulative_gas_used, e.logs); trace!(target: "state", "Transaction receipt: {:?}", receipt); - Ok(ApplyOutcome{receipt: receipt, trace: e.trace}) + + Ok(ApplyOutcome { + receipt, + output, + trace: e.trace, + vm_trace: e.vm_trace, + }) } // Execute a given transaction without committing changes. // // `virt` signals that we are executing outside of a block set and restrictions like // gas limits and gas costs should be lifted. - fn execute(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool, virt: bool) - -> Result + fn execute(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, options: TransactOptions, virt: bool) + -> Result where T: trace::Tracer, V: trace::VMTracer, { - let options = TransactOptions { tracing: tracing, vm_tracing: false, check_nonce: true }; let mut e = Executive::new(self, env_info, engine); match virt { @@ -767,9 +800,8 @@ impl State { Ok(()) } - #[cfg(test)] - #[cfg(feature = "json-tests")] /// Populate the state from `accounts`. + /// Used for tests. pub fn populate_from(&mut self, accounts: PodState) { assert!(self.checkpoints.borrow().is_empty()); for (add, acc) in accounts.drain().into_iter() { diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index 639fce3ab..ea7dd32f5 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use io::IoChannel; use client::{BlockChainClient, MiningBlockChainClient, Client, ClientConfig, BlockId}; use state::{self, State, CleanupMode}; -use executive::Executive; +use executive::{Executive, TransactOptions}; use ethereum; use block::IsBlock; use tests::helpers::*; @@ -361,7 +361,7 @@ fn transaction_proof() { let mut state = State::from_existing(backend, root, 0.into(), factories.clone()).unwrap(); Executive::new(&mut state, &client.latest_env_info(), &*test_spec.engine) - .transact(&transaction, Default::default()).unwrap(); + .transact(&transaction, TransactOptions::with_no_tracing().dont_check_nonce()).unwrap(); assert_eq!(state.balance(&Address::default()).unwrap(), 5.into()); assert_eq!(state.balance(&address).unwrap(), 95.into()); diff --git a/ethcore/src/trace/executive_tracer.rs b/ethcore/src/trace/executive_tracer.rs index 860c74223..ba4d0eff9 100644 --- a/ethcore/src/trace/executive_tracer.rs +++ b/ethcore/src/trace/executive_tracer.rs @@ -167,7 +167,7 @@ impl Tracer for ExecutiveTracer { ExecutiveTracer::default() } - fn traces(self) -> Vec { + fn drain(self) -> Vec { self.traces } } diff --git a/ethcore/src/trace/mod.rs b/ethcore/src/trace/mod.rs index be830430b..c749bfd82 100644 --- a/ethcore/src/trace/mod.rs +++ b/ethcore/src/trace/mod.rs @@ -85,7 +85,7 @@ pub trait Tracer: Send { fn subtracer(&self) -> Self where Self: Sized; /// Consumes self and returns all traces. - fn traces(self) -> Vec; + fn drain(self) -> Vec; } /// Used by executive to build VM traces. diff --git a/ethcore/src/trace/noop_tracer.rs b/ethcore/src/trace/noop_tracer.rs index 2c0e1b830..03d6f57a0 100644 --- a/ethcore/src/trace/noop_tracer.rs +++ b/ethcore/src/trace/noop_tracer.rs @@ -62,7 +62,7 @@ impl Tracer for NoopTracer { NoopTracer } - fn traces(self) -> Vec { + fn drain(self) -> Vec { vec![] } } diff --git a/ethcore/vm/src/tests.rs b/ethcore/vm/src/tests.rs index c8afedaff..c3a290d41 100644 --- a/ethcore/vm/src/tests.rs +++ b/ethcore/vm/src/tests.rs @@ -61,6 +61,7 @@ pub struct FakeExt { pub info: EnvInfo, pub schedule: Schedule, pub balances: HashMap, + pub tracing: bool, } // similar to the normal `finalize` function, but ignoring NeedsReturn. @@ -184,4 +185,8 @@ impl Ext for FakeExt { fn inc_sstore_clears(&mut self) { self.sstore_clears += 1; } + + fn trace_next_instruction(&mut self, _pc: usize, _instruction: u8) -> bool { + self.tracing + } } diff --git a/ethcore/wasm/src/tests.rs b/ethcore/wasm/src/tests.rs index df01b8df6..de0b528a1 100644 --- a/ethcore/wasm/src/tests.rs +++ b/ethcore/wasm/src/tests.rs @@ -415,36 +415,121 @@ fn storage_read() { assert_eq!(Address::from(&result[12..32]), address); } +macro_rules! reqrep_test { + ($name: expr, $input: expr) => { + { + ::ethcore_logger::init_log(); + let code = load_sample!($name); -// Tests that contract's ability to read from a storage -// Test prepopulates address into storage, than executes a contract which read that address from storage and write this address into result + let mut params = ActionParams::default(); + params.gas = U256::from(100_000); + params.code = Some(Arc::new(code)); + params.data = Some($input); + + let (gas_left, result) = { + let mut interpreter = wasm_interpreter(); + let result = interpreter.exec(params, &mut FakeExt::new()).expect("Interpreter to execute without any errors"); + match result { + GasLeft::Known(_) => { panic!("Test is expected to return payload to check"); }, + GasLeft::NeedsReturn { gas_left: gas, data: result, apply_state: _apply } => (gas, result.to_vec()), + } + }; + + (gas_left, result) + } + } +} + +// math_* tests check the ability of wasm contract to perform big integer operations +// - addition +// - multiplication +// - substraction +// - division + +// addition #[test] fn math_add() { - ::ethcore_logger::init_log(); - let code = load_sample!("math.wasm"); - let mut params = ActionParams::default(); - params.gas = U256::from(100_000); - params.code = Some(Arc::new(code)); - - let mut args = [0u8; 64]; - let arg_a = U256::from_dec_str("999999999999999999999999999999").unwrap(); - let arg_b = U256::from_dec_str("888888888888888888888888888888").unwrap(); - arg_a.to_big_endian(&mut args[0..32]); - arg_b.to_big_endian(&mut args[32..64]); - params.data = Some(args.to_vec()); - - let (gas_left, result) = { - let mut interpreter = wasm_interpreter(); - let result = interpreter.exec(params, &mut FakeExt::new()).expect("Interpreter to execute without any errors"); - match result { - GasLeft::Known(_) => { panic!("storage_read should return payload"); }, - GasLeft::NeedsReturn { gas_left: gas, data: result, apply_state: _apply } => (gas, result.to_vec()), + let (gas_left, result) = reqrep_test!( + "math.wasm", + { + let mut args = [0u8; 65]; + let arg_a = U256::from_dec_str("999999999999999999999999999999").unwrap(); + let arg_b = U256::from_dec_str("888888888888888888888888888888").unwrap(); + arg_a.to_big_endian(&mut args[1..33]); + arg_b.to_big_endian(&mut args[33..65]); + args.to_vec() } - }; + ); - let sum: U256 = (&result[..]).into(); - - assert_eq!(gas_left, U256::from(96284)); - assert_eq!(sum, U256::from_dec_str("1888888888888888888888888888887").unwrap()); + assert_eq!(gas_left, U256::from(98087)); + assert_eq!( + U256::from_dec_str("1888888888888888888888888888887").unwrap(), + (&result[..]).into() + ); +} + +// multiplication +#[test] +fn math_mul() { + let (gas_left, result) = reqrep_test!( + "math.wasm", + { + let mut args = [1u8; 65]; + let arg_a = U256::from_dec_str("888888888888888888888888888888").unwrap(); + let arg_b = U256::from_dec_str("999999999999999999999999999999").unwrap(); + arg_a.to_big_endian(&mut args[1..33]); + arg_b.to_big_endian(&mut args[33..65]); + args.to_vec() + } + ); + + assert_eq!(gas_left, U256::from(97236)); + assert_eq!( + U256::from_dec_str("888888888888888888888888888887111111111111111111111111111112").unwrap(), + (&result[..]).into() + ); +} + +// substraction +#[test] +fn math_sub() { + let (gas_left, result) = reqrep_test!( + "math.wasm", + { + let mut args = [2u8; 65]; + let arg_a = U256::from_dec_str("999999999999999999999999999999").unwrap(); + let arg_b = U256::from_dec_str("888888888888888888888888888888").unwrap(); + arg_a.to_big_endian(&mut args[1..33]); + arg_b.to_big_endian(&mut args[33..65]); + args.to_vec() + } + ); + + assert_eq!(gas_left, U256::from(98131)); + assert_eq!( + U256::from_dec_str("111111111111111111111111111111").unwrap(), + (&result[..]).into() + ); +} + +#[test] +fn math_div() { + let (gas_left, result) = reqrep_test!( + "math.wasm", + { + let mut args = [3u8; 65]; + let arg_a = U256::from_dec_str("999999999999999999999999999999").unwrap(); + let arg_b = U256::from_dec_str("888888888888888888888888").unwrap(); + arg_a.to_big_endian(&mut args[1..33]); + arg_b.to_big_endian(&mut args[33..65]); + args.to_vec() + } + ); + + assert_eq!(gas_left, U256::from(91420)); + assert_eq!( + U256::from_dec_str("1125000").unwrap(), + (&result[..]).into() + ); } diff --git a/evmbin/Cargo.toml b/evmbin/Cargo.toml index 698646a3c..5538e10cc 100644 --- a/evmbin/Cargo.toml +++ b/evmbin/Cargo.toml @@ -14,6 +14,7 @@ docopt = "0.8" serde = "1.0" serde_derive = "1.0" ethcore = { path = "../ethcore" } +ethjson = { path = "../json" } ethcore-util = { path = "../util" } evm = { path = "../ethcore/evm" } vm = { path = "../ethcore/vm" } diff --git a/evmbin/src/display/json.rs b/evmbin/src/display/json.rs index e259eec7a..39147ffa3 100644 --- a/evmbin/src/display/json.rs +++ b/evmbin/src/display/json.rs @@ -30,10 +30,8 @@ pub struct Informant { depth: usize, pc: usize, instruction: u8, - name: &'static str, gas_cost: U256, gas_used: U256, - stack_pop: usize, stack: Vec, memory: Vec, storage: HashMap, @@ -58,11 +56,19 @@ impl Informant { } impl vm::Informant for Informant { + fn before_test(&self, name: &str, action: &str) { + println!( + "{{\"test\":\"{name}\",\"action\":\"{action}\"}}", + name = name, + action = action, + ); + } + fn set_gas(&mut self, gas: U256) { self.gas_used = gas; } - fn finish(&mut self, result: Result) { + fn finish(result: Result) { match result { Ok(success) => println!( "{{\"output\":\"0x{output}\",\"gasUsed\":\"{gas:x}\",\"time\":{time}}}", @@ -112,7 +118,7 @@ impl trace::VMTracer for Informant { self.gas_used = gas_used; let len = self.stack.len(); - self.stack.truncate(len - info.args); + self.stack.truncate(if len > info.args { len - info.args } else { 0 }); self.stack.extend_from_slice(stack_push); if let Some((pos, data)) = mem_diff { diff --git a/evmbin/src/display/simple.rs b/evmbin/src/display/simple.rs index b03f97c8d..bb4ecc127 100644 --- a/evmbin/src/display/simple.rs +++ b/evmbin/src/display/simple.rs @@ -27,7 +27,11 @@ use info as vm; pub struct Informant; impl vm::Informant for Informant { - fn finish(&mut self, result: Result) { + fn before_test(&self, name: &str, action: &str) { + println!("Test: {} ({})", name, action); + } + + fn finish(result: Result) { match result { Ok(success) => { println!("Output: 0x{}", success.output.to_hex()); diff --git a/evmbin/src/info.rs b/evmbin/src/info.rs index 617ebc7b9..3392cb441 100644 --- a/evmbin/src/info.rs +++ b/evmbin/src/info.rs @@ -17,17 +17,19 @@ //! VM runner. use std::time::{Instant, Duration}; -use util::U256; -use ethcore::{trace, spec}; -use ethcore::client::{EvmTestClient, EvmTestError}; -use vm::ActionParams; +use util::{U256, H256}; +use ethcore::{trace, spec, transaction, pod_state}; +use ethcore::client::{self, EvmTestClient, EvmTestError, TransactResult}; +use ethjson; /// VM execution informant pub trait Informant: trace::VMTracer { + /// Display a single run init message + fn before_test(&self, test: &str, action: &str); /// Set initial gas. fn set_gas(&mut self, _gas: U256) {} /// Display final result. - fn finish(&mut self, result: Result); + fn finish(result: Result); } /// Execution finished correctly @@ -50,17 +52,71 @@ pub struct Failure { pub time: Duration, } +/// Execute given Transaction and verify resulting state root. +pub fn run_transaction( + name: &str, + idx: usize, + spec: ðjson::state::test::ForkSpec, + pre_state: &pod_state::PodState, + post_root: H256, + env_info: &client::EnvInfo, + transaction: transaction::SignedTransaction, + mut informant: T, +) { + let spec_name = format!("{:?}", spec).to_lowercase(); + let spec = match EvmTestClient::spec_from_json(spec) { + Some(spec) => { + informant.before_test(&format!("{}:{}:{}", name, spec_name, idx), "starting"); + spec + }, + None => { + informant.before_test(&format!("{}:{}:{}", name, spec_name, idx), "skipping because of missing spec"); + return; + }, + }; + + informant.set_gas(env_info.gas_limit); + + let result = run(spec, env_info.gas_limit, pre_state, |mut client| { + let result = client.transact(env_info, transaction, informant); + match result { + TransactResult::Ok { state_root, .. } if state_root != post_root => { + Err(EvmTestError::PostCondition(format!( + "State root mismatch (got: {}, expected: {})", + state_root, + post_root, + ))) + }, + TransactResult::Ok { gas_left, output, .. } => { + Ok((gas_left, output)) + }, + TransactResult::Err { error, .. } => { + Err(EvmTestError::PostCondition(format!( + "Unexpected execution error: {:?}", error + ))) + }, + } + }); + + T::finish(result) +} + /// Execute VM with given `ActionParams` -pub fn run(vm_tracer: &mut T, spec: spec::Spec, params: ActionParams) -> Result { - let mut test_client = EvmTestClient::new(spec).map_err(|error| Failure { +pub fn run<'a, F, T>(spec: &'a spec::Spec, initial_gas: U256, pre_state: T, run: F) -> Result where + F: FnOnce(EvmTestClient) -> Result<(U256, Vec), EvmTestError>, + T: Into>, +{ + let test_client = match pre_state.into() { + Some(pre_state) => EvmTestClient::from_pod_state(spec, pre_state.clone()), + None => EvmTestClient::new(spec), + }.map_err(|error| Failure { gas_used: 0.into(), error, time: Duration::from_secs(0) })?; - let initial_gas = params.gas; let start = Instant::now(); - let result = test_client.call(params, vm_tracer); + let result = run(test_client); let duration = start.elapsed(); match result { diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index bad0ef1f0..5eb43ed61 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -17,8 +17,9 @@ //! Parity EVM interpreter binary. #![warn(missing_docs)] -#![allow(dead_code)] + extern crate ethcore; +extern crate ethjson; extern crate rustc_hex; extern crate serde; #[macro_use] @@ -31,6 +32,7 @@ extern crate panic_hook; use std::sync::Arc; use std::{fmt, fs}; +use std::path::PathBuf; use docopt::Docopt; use rustc_hex::FromHex; use util::{U256, Bytes, Address}; @@ -47,6 +49,7 @@ EVM implementation for Parity. Copyright 2016, 2017 Parity Technologies (UK) Ltd Usage: + parity-evm state-test [--json --only NAME --chain CHAIN] parity-evm stats [options] parity-evm [options] parity-evm [-h | --help] @@ -59,6 +62,10 @@ Transaction options: --gas GAS Supplied gas as hex (without 0x). --gas-price WEI Supplied gas price as hex (without 0x). +State test options: + --only NAME Runs only a single test matching the name. + --chain CHAIN Run only tests from specific chain. + General options: --json Display verbose results in JSON. --chain CHAIN Chain spec file path. @@ -71,20 +78,67 @@ fn main() { let args: Args = Docopt::new(USAGE).and_then(|d| d.deserialize()).unwrap_or_else(|e| e.exit()); - if args.flag_json { - run(args, display::json::Informant::default()) + if args.cmd_state_test { + run_state_test(args) + } else if args.flag_json { + run_call(args, display::json::Informant::default()) } else { - run(args, display::simple::Informant::default()) + run_call(args, display::simple::Informant::default()) } } -fn run(args: Args, mut informant: T) { +fn run_state_test(args: Args) { + use ethjson::state::test::Test; + + let file = args.arg_file.expect("FILE is required"); + let mut file = match fs::File::open(&file) { + Err(err) => die(format!("Unable to open: {:?}: {}", file, err)), + Ok(file) => file, + }; + let state_test = match Test::load(&mut file) { + Err(err) => die(format!("Unable to load the test file: {}", err)), + Ok(test) => test, + }; + let only_test = args.flag_only.map(|s| s.to_lowercase()); + let only_chain = args.flag_chain.map(|s| s.to_lowercase()); + + for (name, test) in state_test { + if let Some(false) = only_test.as_ref().map(|only_test| &name.to_lowercase() == only_test) { + continue; + } + + let multitransaction = test.transaction; + let env_info = test.env.into(); + let pre = test.pre_state.into(); + + for (spec, states) in test.post_states { + if let Some(false) = only_chain.as_ref().map(|only_chain| &format!("{:?}", spec).to_lowercase() == only_chain) { + continue; + } + + for (idx, state) in states.into_iter().enumerate() { + let post_root = state.hash.into(); + let transaction = multitransaction.select(&state.indexes).into(); + + if args.flag_json { + let i = display::json::Informant::default(); + info::run_transaction(&name, idx, &spec, &pre, post_root, &env_info, transaction, i) + } else { + let i = display::simple::Informant::default(); + info::run_transaction(&name, idx, &spec, &pre, post_root, &env_info, transaction, i) + } + } + } + } +} + +fn run_call(args: Args, mut informant: T) { let from = arg(args.from(), "--from"); let to = arg(args.to(), "--to"); let code = arg(args.code(), "--code"); let spec = arg(args.spec(), "--chain"); let gas = arg(args.gas(), "--gas"); - let gas_price = arg(args.gas(), "--gas-price"); + let gas_price = arg(args.gas_price(), "--gas-price"); let data = arg(args.data(), "--input"); if code.is_none() && to == Address::default() { @@ -103,13 +157,18 @@ fn run(args: Args, mut informant: T) { params.data = data; informant.set_gas(gas); - let result = info::run(&mut informant, spec, params); - informant.finish(result); + let result = info::run(&spec, gas, None, |mut client| { + client.call(params, &mut informant) + }); + T::finish(result); } #[derive(Debug, Deserialize)] struct Args { cmd_stats: bool, + cmd_state_test: bool, + arg_file: Option, + flag_only: Option, flag_from: Option, flag_to: Option, flag_code: Option, @@ -221,4 +280,22 @@ mod tests { assert_eq!(args.data(), Ok(Some(vec![06]))); assert_eq!(args.flag_chain, Some("./testfile".to_owned())); } + + #[test] + fn should_parse_state_test_command() { + let args = run(&[ + "parity-evm", + "state-test", + "./file.json", + "--chain", "homestead", + "--only=add11", + "--json", + ]); + + assert_eq!(args.cmd_state_test, true); + assert!(args.arg_file.is_some()); + assert_eq!(args.flag_json, true); + assert_eq!(args.flag_chain, Some("homestead".to_owned())); + assert_eq!(args.flag_only, Some("add11".to_owned())); + } } diff --git a/ipfs/src/route.rs b/ipfs/src/route.rs index b034c2d4c..c72ca53c1 100644 --- a/ipfs/src/route.rs +++ b/ipfs/src/route.rs @@ -119,7 +119,7 @@ mod tests { use ethcore::client::TestBlockChainClient; fn get_mocked_handler() -> IpfsHandler { - IpfsHandler::new(None, None, Arc::new(TestBlockChainClient::new())) + IpfsHandler::new(None.into(), None.into(), Arc::new(TestBlockChainClient::new())) } #[test] diff --git a/js/package-lock.json b/js/package-lock.json index 4e732cbfc..707e5a67d 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,6 +1,6 @@ { "name": "parity.js", - "version": "1.8.16", + "version": "1.8.17", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/js/package.json b/js/package.json index 41388a127..1bb76b21c 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "parity.js", - "version": "1.8.16", + "version": "1.8.17", "main": "release/index.js", "jsnext:main": "src/index.js", "author": "Parity Team ", diff --git a/js/src/api/rpc/parity/parity.js b/js/src/api/rpc/parity/parity.js index c2681b3fb..59ccb5884 100644 --- a/js/src/api/rpc/parity/parity.js +++ b/js/src/api/rpc/parity/parity.js @@ -385,6 +385,11 @@ export default class Parity { .then(outNumber); } + nodeHealth () { + return this._transport + .execute('parity_nodeHealth'); + } + nodeName () { return this._transport .execute('parity_nodeName'); diff --git a/js/src/redux/providers/status.js b/js/src/redux/providers/status.js index e58fcf6c1..39ba74d01 100644 --- a/js/src/redux/providers/status.js +++ b/js/src/redux/providers/status.js @@ -200,7 +200,7 @@ export default class Status { const statusPromises = [ this._api.eth.syncing(), this._api.parity.netPeers(), - this._fetchHealth() + this._api.parity.nodeHealth() ]; return Promise @@ -247,13 +247,6 @@ export default class Status { }; } - _fetchHealth = () => { - // Support Parity-Extension. - const uiUrl = this._api.transport.uiUrlWithProtocol || ''; - - return fetch(`${uiUrl}/api/health`).then(res => res.json()); - } - /** * The data fetched here should not change * unless Parity is restarted. They are thus diff --git a/js/src/views/Account/Header/header.css b/js/src/views/Account/Header/header.css index f894b7c49..5ff69f774 100644 --- a/js/src/views/Account/Header/header.css +++ b/js/src/views/Account/Header/header.css @@ -53,7 +53,7 @@ .infoline, .uuidline { - opacity: 0.25; + opacity: 0.5; } .uuidline { diff --git a/json/src/spec/params.rs b/json/src/spec/params.rs index d72ead89a..de33039c4 100644 --- a/json/src/spec/params.rs +++ b/json/src/spec/params.rs @@ -102,6 +102,9 @@ pub struct Params { pub block_reward: Option, /// See `CommonParams` docs. pub registrar: Option
, + /// Node permission contract address. + #[serde(rename="nodePermissionContract")] + pub node_permission_contract: Option
, } #[cfg(test)] diff --git a/json/src/state/test.rs b/json/src/state/test.rs index ceaccfd17..a63b8ee8d 100644 --- a/json/src/state/test.rs +++ b/json/src/state/test.rs @@ -104,7 +104,10 @@ pub enum ForkSpec { EIP158, Frontier, Homestead, + // TODO [ToDr] Deprecated Metropolis, + Byzantium, + Constantinople, } /// State test indexes deserialization. @@ -161,7 +164,7 @@ mod tests { "EIP150" : [ { "hash" : "3e6dacc1575c6a8c76422255eca03529bbf4c0dda75dfc110b22d6dc4152396f", - "indexes" : { "data" : 0, "gas" : 0, "value" : 0 } + "indexes" : { "data" : 0, "gas" : 0, "value" : 0 } }, { "hash" : "99a450d8ce5b987a71346d8a0a1203711f770745c7ef326912e46761f14cd764", diff --git a/parity/configuration.rs b/parity/configuration.rs index 35a2a7f47..681804346 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -346,6 +346,7 @@ impl Configuration { daemon: daemon, logger_config: logger_config.clone(), miner_options: self.miner_options(self.args.flag_reseal_min_period)?, + ntp_servers: self.ntp_servers(), ws_conf: ws_conf, http_conf: http_conf, ipc_conf: ipc_conf, @@ -563,7 +564,6 @@ impl Configuration { fn ui_config(&self) -> UiConfiguration { UiConfiguration { enabled: self.ui_enabled(), - ntp_servers: self.ntp_servers(), interface: self.ui_interface(), port: self.ui_port(), hosts: self.ui_hosts(), @@ -576,7 +576,6 @@ impl Configuration { DappsConfiguration { enabled: self.dapps_enabled(), - ntp_servers: self.ntp_servers(), dapps_path: PathBuf::from(self.directories().dapps), extra_dapps: if self.args.cmd_dapp { self.args.arg_path.iter().map(|path| PathBuf::from(path)).collect() @@ -1318,12 +1317,6 @@ mod tests { support_token_api: true }, UiConfiguration { enabled: true, - ntp_servers: vec![ - "0.parity.pool.ntp.org:123".into(), - "1.parity.pool.ntp.org:123".into(), - "2.parity.pool.ntp.org:123".into(), - "3.parity.pool.ntp.org:123".into(), - ], interface: "127.0.0.1".into(), port: 8180, hosts: Some(vec![]), @@ -1348,6 +1341,12 @@ mod tests { daemon: None, logger_config: Default::default(), miner_options: Default::default(), + ntp_servers: vec![ + "0.parity.pool.ntp.org:123".into(), + "1.parity.pool.ntp.org:123".into(), + "2.parity.pool.ntp.org:123".into(), + "3.parity.pool.ntp.org:123".into(), + ], ws_conf: Default::default(), http_conf: Default::default(), ipc_conf: Default::default(), @@ -1567,16 +1566,9 @@ mod tests { let conf3 = parse(&["parity", "--ui-path", "signer", "--ui-interface", "test"]); // then - let ntp_servers = vec![ - "0.parity.pool.ntp.org:123".into(), - "1.parity.pool.ntp.org:123".into(), - "2.parity.pool.ntp.org:123".into(), - "3.parity.pool.ntp.org:123".into(), - ]; assert_eq!(conf0.directories().signer, "signer".to_owned()); assert_eq!(conf0.ui_config(), UiConfiguration { enabled: true, - ntp_servers: ntp_servers.clone(), interface: "127.0.0.1".into(), port: 8180, hosts: Some(vec![]), @@ -1585,7 +1577,6 @@ mod tests { assert_eq!(conf1.directories().signer, "signer".to_owned()); assert_eq!(conf1.ui_config(), UiConfiguration { enabled: true, - ntp_servers: ntp_servers.clone(), interface: "127.0.0.1".into(), port: 8180, hosts: Some(vec![]), @@ -1595,7 +1586,6 @@ mod tests { assert_eq!(conf2.directories().signer, "signer".to_owned()); assert_eq!(conf2.ui_config(), UiConfiguration { enabled: true, - ntp_servers: ntp_servers.clone(), interface: "127.0.0.1".into(), port: 3123, hosts: Some(vec![]), @@ -1604,7 +1594,6 @@ mod tests { assert_eq!(conf3.directories().signer, "signer".to_owned()); assert_eq!(conf3.ui_config(), UiConfiguration { enabled: true, - ntp_servers: ntp_servers.clone(), interface: "test".into(), port: 8180, hosts: Some(vec![]), diff --git a/parity/dapps.rs b/parity/dapps.rs index b7ca739ec..98eca3459 100644 --- a/parity/dapps.rs +++ b/parity/dapps.rs @@ -22,12 +22,12 @@ use ethcore::client::{Client, BlockChainClient, BlockId}; use ethcore::transaction::{Transaction, Action}; use ethsync::LightSync; use futures::{future, IntoFuture, Future, BoxFuture}; -use futures_cpupool::CpuPool; use hash_fetch::fetch::Client as FetchClient; use hash_fetch::urlhint::ContractClient; use helpers::replace_home; use light::client::LightChainClient; use light::on_demand::{self, OnDemand}; +use node_health::{SyncStatus, NodeHealth}; use rpc; use rpc_apis::SignerService; use parity_reactor; @@ -36,7 +36,6 @@ use util::{Bytes, Address}; #[derive(Debug, PartialEq, Clone)] pub struct Configuration { pub enabled: bool, - pub ntp_servers: Vec, pub dapps_path: PathBuf, pub extra_dapps: Vec, pub extra_embed_on: Vec<(String, u16)>, @@ -48,12 +47,6 @@ impl Default for Configuration { let data_dir = default_data_path(); Configuration { enabled: true, - ntp_servers: vec![ - "0.parity.pool.ntp.org:123".into(), - "1.parity.pool.ntp.org:123".into(), - "2.parity.pool.ntp.org:123".into(), - "3.parity.pool.ntp.org:123".into(), - ], dapps_path: replace_home(&data_dir, "$BASE/dapps").into(), extra_dapps: vec![], extra_embed_on: vec![], @@ -156,10 +149,10 @@ impl ContractClient for LightRegistrar { // to resolve. #[derive(Clone)] pub struct Dependencies { + pub node_health: NodeHealth, pub sync_status: Arc, pub contract_client: Arc, pub remote: parity_reactor::TokioRemote, - pub pool: CpuPool, pub fetch: FetchClient, pub signer: Arc, pub ui_address: Option<(String, u16)>, @@ -172,7 +165,6 @@ pub fn new(configuration: Configuration, deps: Dependencies) -> Result Result Result, String> { +pub fn new_ui(enabled: bool, deps: Dependencies) -> Result, String> { if !enabled { return Ok(None); } server::ui_middleware( deps, - ntp_servers, rpc::DAPPS_DOMAIN, ).map(Some) } -pub use self::server::{SyncStatus, Middleware, service}; +pub use self::server::{Middleware, service}; #[cfg(not(feature = "dapps"))] mod server { @@ -203,11 +194,6 @@ mod server { use parity_rpc::{hyper, RequestMiddleware, RequestMiddlewareAction}; use rpc_apis; - pub trait SyncStatus { - fn is_major_importing(&self) -> bool; - fn peers(&self) -> (usize, usize); - } - pub struct Middleware; impl RequestMiddleware for Middleware { fn on_request( @@ -219,7 +205,6 @@ mod server { pub fn dapps_middleware( _deps: Dependencies, - _ntp_servers: &[String], _dapps_path: PathBuf, _extra_dapps: Vec, _dapps_domain: &str, @@ -231,7 +216,6 @@ mod server { pub fn ui_middleware( _deps: Dependencies, - _ntp_servers: &[String], _dapps_domain: &str, ) -> Result { Err("Your Parity version has been compiled without UI support.".into()) @@ -253,11 +237,9 @@ mod server { use parity_reactor; pub use parity_dapps::Middleware; - pub use parity_dapps::SyncStatus; pub fn dapps_middleware( deps: Dependencies, - ntp_servers: &[String], dapps_path: PathBuf, extra_dapps: Vec, dapps_domain: &str, @@ -269,8 +251,7 @@ mod server { let web_proxy_tokens = Arc::new(move |token| signer.web_proxy_access_token_domain(&token)); Ok(parity_dapps::Middleware::dapps( - ntp_servers, - deps.pool, + deps.node_health, parity_remote, deps.ui_address, extra_embed_on, @@ -287,13 +268,11 @@ mod server { pub fn ui_middleware( deps: Dependencies, - ntp_servers: &[String], dapps_domain: &str, ) -> Result { let parity_remote = parity_reactor::Remote::new(deps.remote.clone()); Ok(parity_dapps::Middleware::ui( - ntp_servers, - deps.pool, + deps.node_health, parity_remote, dapps_domain, deps.contract_client, diff --git a/parity/main.rs b/parity/main.rs index 48e752bf2..d79fb0064 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -58,6 +58,7 @@ extern crate ethcore_util as util; extern crate ethcore_network as network; extern crate ethkey; extern crate ethsync; +extern crate node_health; extern crate panic_hook; extern crate parity_hash_fetch as hash_fetch; extern crate parity_ipfs_api; @@ -68,6 +69,7 @@ extern crate parity_updater as updater; extern crate parity_whisper; extern crate path; extern crate rpc_cli; +extern crate node_filter; #[macro_use] extern crate log as rlog; diff --git a/parity/modules.rs b/parity/modules.rs index c7aea7dfa..8613c5ae5 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -19,7 +19,7 @@ use std::path::Path; use ethcore::client::BlockChainClient; use hypervisor::Hypervisor; -use ethsync::{AttachedProtocol, SyncConfig, NetworkConfiguration, NetworkError, Params}; +use ethsync::{AttachedProtocol, SyncConfig, NetworkConfiguration, NetworkError, Params, ConnectionFilter}; use ethcore::snapshot::SnapshotService; use light::Provider; @@ -183,6 +183,7 @@ pub fn sync( provider: Arc, _log_settings: &LogConfig, attached_protos: Vec, + connection_filter: Option>, ) -> Result { let eth_sync = EthSync::new(Params { config: sync_cfg, @@ -191,7 +192,8 @@ pub fn sync( snapshot_service: snapshot_service, network_config: net_cfg, attached_protos: attached_protos, - })?; + }, + connection_filter)?; Ok((eth_sync.clone() as Arc, eth_sync.clone() as Arc, eth_sync.clone() as Arc)) } diff --git a/parity/rpc.rs b/parity/rpc.rs index 1e148dad2..f11d4c3ae 100644 --- a/parity/rpc.rs +++ b/parity/rpc.rs @@ -73,7 +73,6 @@ impl Default for HttpConfiguration { #[derive(Debug, PartialEq, Clone)] pub struct UiConfiguration { pub enabled: bool, - pub ntp_servers: Vec, pub interface: String, pub port: u16, pub hosts: Option>, @@ -107,12 +106,6 @@ impl Default for UiConfiguration { fn default() -> Self { UiConfiguration { enabled: true && cfg!(feature = "ui-enabled"), - ntp_servers: vec![ - "0.parity.pool.ntp.org:123".into(), - "1.parity.pool.ntp.org:123".into(), - "2.parity.pool.ntp.org:123".into(), - "3.parity.pool.ntp.org:123".into(), - ], port: 8180, interface: "127.0.0.1".into(), hosts: Some(vec![]), diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index bb1e1c377..705006a9e 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -27,18 +27,19 @@ use ethcore::account_provider::AccountProvider; use ethcore::client::Client; use ethcore::miner::{Miner, ExternalMiner}; use ethcore::snapshot::SnapshotService; -use parity_rpc::{Metadata, NetworkSettings}; -use parity_rpc::informant::{ActivityNotifier, ClientNotifier}; -use parity_rpc::dispatch::{FullDispatcher, LightDispatcher}; +use ethcore_logger::RotatingLogger; use ethsync::{ManageNetwork, SyncProvider, LightSync}; use hash_fetch::fetch::Client as FetchClient; use jsonrpc_core::{self as core, MetaIoHandler}; use light::{TransactionQueue as LightTransactionQueue, Cache as LightDataCache}; use light::client::LightChainClient; +use node_health::NodeHealth; +use parity_reactor; +use parity_rpc::dispatch::{FullDispatcher, LightDispatcher}; +use parity_rpc::informant::{ActivityNotifier, ClientNotifier}; +use parity_rpc::{Metadata, NetworkSettings}; use updater::Updater; use util::{Mutex, RwLock}; -use ethcore_logger::RotatingLogger; -use parity_reactor; #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum Api { @@ -218,6 +219,7 @@ pub struct FullDependencies { pub settings: Arc, pub net_service: Arc, pub updater: Arc, + pub health: NodeHealth, pub geth_compatibility: bool, pub dapps_service: Option>, pub dapps_address: Option<(String, u16)>, @@ -301,12 +303,13 @@ impl FullDependencies { false => None, }; handler.extend_with(ParityClient::new( - &self.client, - &self.miner, - &self.sync, - &self.updater, - &self.net_service, - &self.secret_store, + self.client.clone(), + self.miner.clone(), + self.sync.clone(), + self.updater.clone(), + self.net_service.clone(), + self.health.clone(), + self.secret_store.clone(), self.logger.clone(), self.settings.clone(), signer, @@ -404,6 +407,7 @@ pub struct LightDependencies { pub secret_store: Arc, pub logger: Arc, pub settings: Arc, + pub health: NodeHealth, pub on_demand: Arc<::light::on_demand::OnDemand>, pub cache: Arc>, pub transaction_queue: Arc>, @@ -508,6 +512,7 @@ impl LightDependencies { self.secret_store.clone(), self.logger.clone(), self.settings.clone(), + self.health.clone(), signer, self.dapps_address.clone(), self.ws_address.clone(), diff --git a/parity/run.rs b/parity/run.rs index 277ee9025..a233055c1 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::fmt; use std::sync::{Arc, Weak}; use std::net::{TcpListener}; @@ -32,10 +33,12 @@ use fdlimit::raise_fd_limit; use hash_fetch::fetch::{Fetch, Client as FetchClient}; use informant::{Informant, LightNodeInformantData, FullNodeInformantData}; use light::Cache as LightDataCache; +use node_health; use parity_reactor::EventLoop; use parity_rpc::{NetworkSettings, informant, is_major_importing}; use updater::{UpdatePolicy, Updater}; use util::{Colour, version, Mutex, Condvar}; +use node_filter::NodeFilter; use params::{ SpecType, Pruning, AccountsConfig, GasPricerConfig, MinerExtras, Switch, @@ -80,6 +83,7 @@ pub struct RunCmd { pub daemon: Option, pub logger_config: LogConfig, pub miner_options: MinerOptions, + pub ntp_servers: Vec, pub ws_conf: rpc::WsConfiguration, pub http_conf: rpc::HttpConfiguration, pub ipc_conf: rpc::IpcConfiguration, @@ -290,7 +294,7 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc) -> // the dapps server let signer_service = Arc::new(signer::new_service(&cmd.ws_conf, &cmd.ui_conf, &cmd.logger_config)); - let dapps_deps = { + let (node_health, dapps_deps) = { let contract_client = Arc::new(::dapps::LightRegistrar { client: service.client().clone(), sync: light_sync.clone(), @@ -298,7 +302,12 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc) -> }); struct LightSyncStatus(Arc); - impl dapps::SyncStatus for LightSyncStatus { + impl fmt::Debug for LightSyncStatus { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Light Sync Status") + } + } + impl node_health::SyncStatus for LightSyncStatus { fn is_major_importing(&self) -> bool { self.0.is_major_importing() } fn peers(&self) -> (usize, usize) { let peers = ethsync::LightSyncProvider::peer_numbers(&*self.0); @@ -306,19 +315,26 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc) -> } } - dapps::Dependencies { - sync_status: Arc::new(LightSyncStatus(light_sync.clone())), - pool: fetch.pool(), + let sync_status = Arc::new(LightSyncStatus(light_sync.clone())); + let node_health = node_health::NodeHealth::new( + sync_status.clone(), + node_health::TimeChecker::new(&cmd.ntp_servers, fetch.pool()), + event_loop.remote(), + ); + + (node_health.clone(), dapps::Dependencies { + sync_status, + node_health, contract_client: contract_client, remote: event_loop.raw_remote(), fetch: fetch.clone(), signer: signer_service.clone(), ui_address: cmd.ui_conf.address(), - } + }) }; let dapps_middleware = dapps::new(cmd.dapps_conf.clone(), dapps_deps.clone())?; - let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, &cmd.ui_conf.ntp_servers, dapps_deps)?; + let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, dapps_deps)?; // start RPCs let dapps_service = dapps::service(&dapps_middleware); @@ -327,6 +343,7 @@ fn execute_light(cmd: RunCmd, can_restart: bool, logger: Arc) -> client: service.client().clone(), sync: light_sync.clone(), net: light_sync.clone(), + health: node_health, secret_store: account_provider, logger: logger, settings: Arc::new(cmd.net_settings), @@ -560,11 +577,13 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc) -> R miner.clone(), ).map_err(|e| format!("Client service error: {:?}", e))?; + let connection_filter_address = spec.params().node_permission_contract; // drop the spec to free up genesis state. drop(spec); // take handle to client let client = service.client(); + let connection_filter = connection_filter_address.map(|a| Arc::new(NodeFilter::new(Arc::downgrade(&client) as Weak, a))); let snapshot_service = service.snapshot_service(); // initialize the local node information store. @@ -636,9 +655,13 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc) -> R client.clone(), &cmd.logger_config, attached_protos, + connection_filter.clone().map(|f| f as Arc<::ethsync::ConnectionFilter + 'static>), ).map_err(|e| format!("Sync error: {}", e))?; service.add_notify(chain_notify.clone()); + if let Some(filter) = connection_filter { + service.add_notify(filter); + } // start network if network_enabled { @@ -668,12 +691,17 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc) -> R let signer_service = Arc::new(signer::new_service(&cmd.ws_conf, &cmd.ui_conf, &cmd.logger_config)); // the dapps server - let dapps_deps = { + let (node_health, dapps_deps) = { let (sync, client) = (sync_provider.clone(), client.clone()); let contract_client = Arc::new(::dapps::FullRegistrar { client: client.clone() }); struct SyncStatus(Arc, Arc, ethsync::NetworkConfiguration); - impl dapps::SyncStatus for SyncStatus { + impl fmt::Debug for SyncStatus { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Dapps Sync Status") + } + } + impl node_health::SyncStatus for SyncStatus { fn is_major_importing(&self) -> bool { is_major_importing(Some(self.0.status().state), self.1.queue_info()) } @@ -683,18 +711,24 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc) -> R } } - dapps::Dependencies { - sync_status: Arc::new(SyncStatus(sync, client, net_conf)), - pool: fetch.pool(), + let sync_status = Arc::new(SyncStatus(sync, client, net_conf)); + let node_health = node_health::NodeHealth::new( + sync_status.clone(), + node_health::TimeChecker::new(&cmd.ntp_servers, fetch.pool()), + event_loop.remote(), + ); + (node_health.clone(), dapps::Dependencies { + sync_status, + node_health, contract_client: contract_client, remote: event_loop.raw_remote(), fetch: fetch.clone(), signer: signer_service.clone(), ui_address: cmd.ui_conf.address(), - } + }) }; let dapps_middleware = dapps::new(cmd.dapps_conf.clone(), dapps_deps.clone())?; - let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, &cmd.ui_conf.ntp_servers, dapps_deps)?; + let ui_middleware = dapps::new_ui(cmd.ui_conf.enabled, dapps_deps)?; let dapps_service = dapps::service(&dapps_middleware); let deps_for_rpc_apis = Arc::new(rpc_apis::FullDependencies { @@ -702,6 +736,7 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc) -> R snapshot: snapshot_service.clone(), client: client.clone(), sync: sync_provider.clone(), + health: node_health, net: manage_network.clone(), secret_store: secret_store, miner: miner.clone(), diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 73230c24e..a5a371cdf 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -47,12 +47,13 @@ ethjson = { path = "../json" } ethcore-devtools = { path = "../devtools" } ethcore-light = { path = "../ethcore/light" } ethcore-logger = { path = "../logger" } -vm = { path = "../ethcore/vm" } -parity-updater = { path = "../updater" } -parity-reactor = { path = "../util/reactor" } -rlp = { path = "../util/rlp" } fetch = { path = "../util/fetch" } +node-health = { path = "../dapps/node-health" } +parity-reactor = { path = "../util/reactor" } +parity-updater = { path = "../updater" } +rlp = { path = "../util/rlp" } stats = { path = "../util/stats" } +vm = { path = "../ethcore/vm" } clippy = { version = "0.0.103", optional = true} pretty_assertions = "0.1" diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index e75d187de..c429471f3 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -55,6 +55,7 @@ extern crate ethsync; extern crate ethcore_logger; extern crate vm; extern crate fetch; +extern crate node_health; extern crate parity_reactor; extern crate parity_updater as updater; extern crate rlp; diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index b2602d6a1..ab8dd655a 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -584,8 +584,9 @@ fn decrypt(accounts: &AccountProvider, address: Address, msg: Bytes, password: S } /// Extract the default gas price from a client and miner. -pub fn default_gas_price(client: &C, miner: &M) -> U256 - where C: MiningBlockChainClient, M: MinerService +pub fn default_gas_price(client: &C, miner: &M) -> U256 where + C: MiningBlockChainClient, + M: MinerService, { client.gas_price_corpus(100).median().cloned().unwrap_or_else(|| miner.sensible_gas_price()) } diff --git a/rpc/src/v1/helpers/fake_sign.rs b/rpc/src/v1/helpers/fake_sign.rs index c5b95f6a4..2bbaef0ee 100644 --- a/rpc/src/v1/helpers/fake_sign.rs +++ b/rpc/src/v1/helpers/fake_sign.rs @@ -24,21 +24,23 @@ use jsonrpc_core::Error; use v1::helpers::CallRequest; use v1::helpers::dispatch::default_gas_price; -pub fn sign_call( - client: &Arc, +pub fn sign_call ( + client: &Arc, miner: &Arc, request: CallRequest, gas_cap: bool, ) -> Result { - let from = request.from.unwrap_or(0.into()); - let mut gas = request.gas.unwrap_or(U256::max_value()); - if gas_cap { - let max_gas = 50_000_000.into(); - if gas > max_gas { + let max_gas = 50_000_000.into(); + let gas = match request.gas { + Some(gas) if gas_cap && gas > max_gas => { warn!("Gas limit capped to {} (from {})", max_gas, gas); - gas = max_gas + max_gas } - } + Some(gas) => gas, + None if gas_cap => max_gas, + None => U256::from(2) << 50, + }; + let from = request.from.unwrap_or(0.into()); Ok(Transaction { nonce: request.nonce.unwrap_or_else(|| client.latest_nonce(&from)), diff --git a/rpc/src/v1/impls/light/parity.rs b/rpc/src/v1/impls/light/parity.rs index 676e39b11..7cd540937 100644 --- a/rpc/src/v1/impls/light/parity.rs +++ b/rpc/src/v1/impls/light/parity.rs @@ -19,15 +19,15 @@ use std::sync::Arc; use std::collections::{BTreeMap, HashSet}; use futures::{future, Future, BoxFuture}; -use ethcore_logger::RotatingLogger; use util::misc::version_data; -use crypto::ecies; +use crypto::{ecies, DEFAULT_MAC}; use ethkey::{Brain, Generator}; use ethstore::random_phrase; use ethsync::LightSyncProvider; use ethcore::account_provider::AccountProvider; -use crypto::DEFAULT_MAC; +use ethcore_logger::RotatingLogger; +use node_health::{NodeHealth, Health}; use light::client::LightChainClient; @@ -53,6 +53,7 @@ pub struct ParityClient { accounts: Arc, logger: Arc, settings: Arc, + health: NodeHealth, signer: Option>, dapps_address: Option<(String, u16)>, ws_address: Option<(String, u16)>, @@ -67,18 +68,20 @@ impl ParityClient { accounts: Arc, logger: Arc, settings: Arc, + health: NodeHealth, signer: Option>, dapps_address: Option<(String, u16)>, ws_address: Option<(String, u16)>, ) -> Self { ParityClient { - light_dispatch: light_dispatch, - accounts: accounts, - logger: logger, - settings: settings, - signer: signer, - dapps_address: dapps_address, - ws_address: ws_address, + light_dispatch, + accounts, + logger, + settings, + health, + signer, + dapps_address, + ws_address, eip86_transition: client.eip86_transition(), } } @@ -393,4 +396,10 @@ impl Parity for ParityClient { fn call(&self, _meta: Self::Metadata, _requests: Vec, _block: Trailing) -> BoxFuture, Error> { future::err(errors::light_unimplemented(None)).boxed() } + + fn node_health(&self) -> BoxFuture { + self.health.health() + .map_err(|err| errors::internal("Health API failure.", err)) + .boxed() + } } diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index fb82c5205..e66d3ac05 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -20,11 +20,10 @@ use std::str::FromStr; use std::collections::{BTreeMap, HashSet}; use futures::{future, Future, BoxFuture}; -use ethcore_logger::RotatingLogger; use util::Address; use util::misc::version_data; -use crypto::ecies; +use crypto::{DEFAULT_MAC, ecies}; use ethkey::{Brain, Generator}; use ethstore::random_phrase; use ethsync::{SyncProvider, ManageNetwork}; @@ -34,8 +33,9 @@ use ethcore::ids::BlockId; use ethcore::miner::MinerService; use ethcore::mode::Mode; use ethcore::transaction::SignedTransaction; +use ethcore_logger::RotatingLogger; +use node_health::{NodeHealth, Health}; use updater::{Service as UpdateService}; -use crypto::DEFAULT_MAC; use jsonrpc_core::Error; use jsonrpc_macros::Trailing; @@ -53,17 +53,13 @@ use v1::types::{ }; /// Parity implementation. -pub struct ParityClient where - C: MiningBlockChainClient, - M: MinerService, - S: SyncProvider, - U: UpdateService, -{ +pub struct ParityClient { client: Arc, miner: Arc, - sync: Arc, updater: Arc, + sync: Arc, net: Arc, + health: NodeHealth, accounts: Option>, logger: Arc, settings: Arc, @@ -73,39 +69,39 @@ pub struct ParityClient where eip86_transition: u64, } -impl ParityClient where +impl ParityClient where C: MiningBlockChainClient, - M: MinerService, - S: SyncProvider, - U: UpdateService, { /// Creates new `ParityClient`. pub fn new( - client: &Arc, - miner: &Arc, - sync: &Arc, - updater: &Arc, - net: &Arc, - store: &Option>, + client: Arc, + miner: Arc, + sync: Arc, + updater: Arc, + net: Arc, + health: NodeHealth, + accounts: Option>, logger: Arc, settings: Arc, signer: Option>, dapps_address: Option<(String, u16)>, ws_address: Option<(String, u16)>, ) -> Self { + let eip86_transition = client.eip86_transition(); ParityClient { - client: client.clone(), - miner: miner.clone(), - sync: sync.clone(), - updater: updater.clone(), - net: net.clone(), - accounts: store.clone(), - logger: logger, - settings: settings, - signer: signer, - dapps_address: dapps_address, - ws_address: ws_address, - eip86_transition: client.eip86_transition(), + client, + miner, + sync, + updater, + net, + health, + accounts, + logger, + settings, + signer, + dapps_address, + ws_address, + eip86_transition, } } @@ -116,10 +112,9 @@ impl ParityClient where } } -impl Parity for ParityClient where - M: MinerService + 'static, +impl Parity for ParityClient where C: MiningBlockChainClient + 'static, - S: SyncProvider + 'static, + M: MinerService + 'static, U: UpdateService + 'static, { type Metadata = Metadata; @@ -429,4 +424,10 @@ impl Parity for ParityClient where future::done(result).boxed() } + + fn node_health(&self) -> BoxFuture { + self.health.health() + .map_err(|err| errors::internal("Health API failure.", err)) + .boxed() + } } diff --git a/rpc/src/v1/tests/mocked/parity.rs b/rpc/src/v1/tests/mocked/parity.rs index b5cdada6f..3c98ea8ef 100644 --- a/rpc/src/v1/tests/mocked/parity.rs +++ b/rpc/src/v1/tests/mocked/parity.rs @@ -15,13 +15,15 @@ // along with Parity. If not, see . use std::sync::Arc; -use ethcore_logger::RotatingLogger; -use util::{Address, U256}; -use ethsync::ManageNetwork; use ethcore::account_provider::AccountProvider; use ethcore::client::{TestBlockChainClient, Executed}; use ethcore::miner::LocalTransactionStatus; +use ethcore_logger::RotatingLogger; use ethstore::ethkey::{Generator, Random}; +use ethsync::ManageNetwork; +use node_health::{self, NodeHealth}; +use parity_reactor; +use util::Address; use jsonrpc_core::IoHandler; use v1::{Parity, ParityClient}; @@ -30,13 +32,14 @@ use v1::helpers::{SignerService, NetworkSettings}; use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService, TestUpdater}; use super::manage_network::TestManageNetwork; -pub type TestParityClient = ParityClient; +pub type TestParityClient = ParityClient; pub struct Dependencies { pub miner: Arc, pub client: Arc, pub sync: Arc, pub updater: Arc, + pub health: NodeHealth, pub logger: Arc, pub settings: Arc, pub network: Arc, @@ -54,6 +57,11 @@ impl Dependencies { network_id: 3, num_peers: 120, })), + health: NodeHealth::new( + Arc::new(FakeSync), + node_health::TimeChecker::new::(&[], node_health::CpuPool::new(1)), + parity_reactor::Remote::new_sync(), + ), updater: Arc::new(TestUpdater::default()), logger: Arc::new(RotatingLogger::new("rpc=trace".to_owned())), settings: Arc::new(NetworkSettings { @@ -75,12 +83,13 @@ impl Dependencies { let opt_accounts = Some(self.accounts.clone()); ParityClient::new( - &self.client, - &self.miner, - &self.sync, - &self.updater, - &self.network, - &opt_accounts, + self.client.clone(), + self.miner.clone(), + self.sync.clone(), + self.updater.clone(), + self.network.clone(), + self.health.clone(), + opt_accounts.clone(), self.logger.clone(), self.settings.clone(), signer, @@ -102,6 +111,13 @@ impl Dependencies { } } +#[derive(Debug)] +struct FakeSync; +impl node_health::SyncStatus for FakeSync { + fn is_major_importing(&self) -> bool { false } + fn peers(&self) -> (usize, usize) { (4, 25) } +} + #[test] fn rpc_parity_accounts_info() { let deps = Dependencies::new(); @@ -507,6 +523,8 @@ fn rpc_parity_cid() { #[test] fn rpc_parity_call() { + use util::U256; + let deps = Dependencies::new(); deps.client.set_execution_result(Ok(Executed { exception: None, @@ -541,3 +559,14 @@ fn rpc_parity_call() { assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); } + +#[test] +fn rpc_parity_node_health() { + let deps = Dependencies::new(); + let io = deps.default_client(); + + let request = r#"{"jsonrpc": "2.0", "method": "parity_nodeHealth", "params":[], "id": 1}"#; + let response = r#"{"jsonrpc":"2.0","result":{"peers":{"details":[4,25],"message":"","status":"ok"},"sync":{"details":false,"message":"","status":"ok"},"time":{"details":0,"message":"","status":"ok"}},"id":1}"#; + + assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); +} diff --git a/rpc/src/v1/traits/parity.rs b/rpc/src/v1/traits/parity.rs index c9ca522c1..f1512b8ce 100644 --- a/rpc/src/v1/traits/parity.rs +++ b/rpc/src/v1/traits/parity.rs @@ -22,6 +22,7 @@ use jsonrpc_core::Error; use jsonrpc_macros::Trailing; use futures::BoxFuture; +use node_health::Health; use v1::types::{ H160, H256, H512, U256, Bytes, CallRequest, Peers, Transaction, RpcSettings, Histogram, @@ -207,5 +208,9 @@ build_rpc_trait! { /// Call contract, returning the output data. #[rpc(meta, name = "parity_call")] fn call(&self, Self::Metadata, Vec, Trailing) -> BoxFuture, Error>; + + /// Returns node's health report. + #[rpc(async, name = "parity_nodeHealth")] + fn node_health(&self) -> BoxFuture; } } diff --git a/sync/src/api.rs b/sync/src/api.rs index acbc26867..f3c5570ee 100644 --- a/sync/src/api.rs +++ b/sync/src/api.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap, BTreeMap}; use std::io; use util::Bytes; use network::{NetworkProtocolHandler, NetworkService, NetworkContext, HostInfo, PeerId, ProtocolId, - NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, NetworkError}; + NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, NetworkError, ConnectionFilter}; use util::{U256, H256, H512}; use io::{TimerToken}; use ethcore::ethstore::ethkey::Secret; @@ -236,7 +236,7 @@ pub struct EthSync { impl EthSync { /// Creates and register protocol with the network service - pub fn new(params: Params) -> Result, NetworkError> { + pub fn new(params: Params, connection_filter: Option>) -> Result, NetworkError> { const MAX_LIGHTSERV_LOAD: f64 = 0.5; let pruning_info = params.chain.pruning_info(); @@ -272,7 +272,7 @@ impl EthSync { }; let chain_sync = ChainSync::new(params.config, &*params.chain); - let service = NetworkService::new(params.network_config.clone().into_basic()?)?; + let service = NetworkService::new(params.network_config.clone().into_basic()?, connection_filter)?; let sync = Arc::new(EthSync { network: service, @@ -736,7 +736,7 @@ impl LightSync { (sync_handler, Arc::new(light_proto)) }; - let service = NetworkService::new(params.network_config)?; + let service = NetworkService::new(params.network_config, None)?; Ok(LightSync { proto: light_proto, diff --git a/sync/src/lib.rs b/sync/src/lib.rs index e131bf901..51947317d 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -76,7 +76,7 @@ mod api; pub use api::*; pub use chain::{SyncStatus, SyncState}; -pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError}; +pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError, ConnectionFilter, ConnectionDirection}; /// IPC interfaces #[cfg(feature="ipc")] diff --git a/test.sh b/test.sh index 5430b17ea..7a2cac946 100755 --- a/test.sh +++ b/test.sh @@ -26,4 +26,4 @@ set -e ./scripts/validate_chainspecs.sh -cargo test -j 8 $OPTIONS --features "$FEATURES" --all --exclude parity-ipfs-api --exclude evmjit $1 +cargo test -j 8 $OPTIONS --features "$FEATURES" --all --exclude evmjit $1 diff --git a/util/bigint/Cargo.toml b/util/bigint/Cargo.toml index 36a272f8c..fcca56078 100644 --- a/util/bigint/Cargo.toml +++ b/util/bigint/Cargo.toml @@ -13,6 +13,7 @@ rustc-hex = "1.0" rand = "0.3.12" libc = "0.2" heapsize = { version = "0.4", optional = true } +plain_hasher = { path = "../plain_hasher" } [features] x64asm_arithmetic=[] diff --git a/util/bigint/src/hash.rs b/util/bigint/src/hash.rs index ca6adf0e7..316062bc4 100644 --- a/util/bigint/src/hash.rs +++ b/util/bigint/src/hash.rs @@ -16,6 +16,7 @@ use std::collections::{HashMap, HashSet}; use rand::{Rand, Rng}; use rand::os::OsRng; use rustc_hex::{FromHex, FromHexError}; +use plain_hasher::PlainHasher; use bigint::U256; use libc::{c_void, memcmp}; @@ -446,41 +447,6 @@ impl_hash!(H2048, 256); known_heap_size!(0, H32, H64, H128, H160, H256, H264, H512, H520, H1024, H2048); // Specialized HashMap and HashSet -/// Hasher that just takes 8 bytes of the provided value. -/// May only be used for keys which are 32 bytes. -pub struct PlainHasher { - prefix: [u8; 8], - _marker: [u64; 0], // for alignment -} - -impl Default for PlainHasher { - #[inline] - fn default() -> PlainHasher { - PlainHasher { - prefix: [0; 8], - _marker: [0; 0], - } - } -} - -impl Hasher for PlainHasher { - #[inline] - fn finish(&self) -> u64 { - unsafe { ::std::mem::transmute(self.prefix) } - } - - #[inline] - fn write(&mut self, bytes: &[u8]) { - debug_assert!(bytes.len() == 32); - - for quarter in bytes.chunks(8) { - for (x, y) in self.prefix.iter_mut().zip(quarter) { - *x ^= *y - } - } - } -} - /// Specialized version of `HashMap` with H256 keys and fast hashing function. pub type H256FastMap = HashMap>; /// Specialized version of `HashSet` with H256 keys and fast hashing function. diff --git a/util/bigint/src/lib.rs b/util/bigint/src/lib.rs index 7a85de6a1..1b1b5d488 100644 --- a/util/bigint/src/lib.rs +++ b/util/bigint/src/lib.rs @@ -14,6 +14,7 @@ extern crate rand; extern crate rustc_hex; extern crate bigint; extern crate libc; +extern crate plain_hasher; #[cfg(feature="heapsizeof")] #[macro_use] diff --git a/util/bloom/src/lib.rs b/util/bloom/src/lib.rs index 16e0e5368..91897ed44 100644 --- a/util/bloom/src/lib.rs +++ b/util/bloom/src/lib.rs @@ -37,9 +37,9 @@ struct BitVecJournal { impl BitVecJournal { pub fn new(size: usize) -> BitVecJournal { - let extra = if size % 8 > 0 { 1 } else { 0 }; + let extra = if size % 64 > 0 { 1 } else { 0 }; BitVecJournal { - elems: vec![0u64; size / 8 + extra], + elems: vec![0u64; size / 64 + extra], journal: HashSet::new(), } } diff --git a/util/network/src/connection_filter.rs b/util/network/src/connection_filter.rs new file mode 100644 index 000000000..5afe5865b --- /dev/null +++ b/util/network/src/connection_filter.rs @@ -0,0 +1,31 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Connection filter trait. + +use super::NodeId; + +/// Filtered connection direction. +pub enum ConnectionDirection { + Inbound, + Outbound, +} + +/// Connection filter. Each connection is checked against `connection_allowed`. +pub trait ConnectionFilter : Send + Sync { + /// Filter a connection. Returns `true` if connection should be allowed. `false` if rejected. + fn connection_allowed(&self, own_id: &NodeId, connecting_id: &NodeId, direction: ConnectionDirection) -> bool; +} diff --git a/util/network/src/host.rs b/util/network/src/host.rs index 8aea9184f..d74b2fa6e 100644 --- a/util/network/src/host.rs +++ b/util/network/src/host.rs @@ -42,6 +42,7 @@ use discovery::{Discovery, TableUpdates, NodeEntry}; use ip_utils::{map_external_address, select_public_address}; use path::restrict_permissions_owner; use parking_lot::{Mutex, RwLock}; +use connection_filter::{ConnectionFilter, ConnectionDirection}; type Slab = ::slab::Slab; @@ -380,11 +381,12 @@ pub struct Host { reserved_nodes: RwLock>, num_sessions: AtomicUsize, stopping: AtomicBool, + filter: Option>, } impl Host { /// Create a new instance - pub fn new(mut config: NetworkConfiguration, stats: Arc) -> Result { + pub fn new(mut config: NetworkConfiguration, stats: Arc, filter: Option>) -> Result { let mut listen_address = match config.listen_address { None => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(0, 0, 0, 0), DEFAULT_PORT)), Some(addr) => addr, @@ -437,6 +439,7 @@ impl Host { reserved_nodes: RwLock::new(HashSet::new()), num_sessions: AtomicUsize::new(0), stopping: AtomicBool::new(false), + filter: filter, }; for n in boot_nodes { @@ -691,8 +694,12 @@ impl Host { let max_handshakes_per_round = max_handshakes / 2; let mut started: usize = 0; - for id in nodes.filter(|id| !self.have_session(id) && !self.connecting_to(id) && *id != self_id) - .take(min(max_handshakes_per_round, max_handshakes - handshake_count)) { + for id in nodes.filter(|id| + !self.have_session(id) && + !self.connecting_to(id) && + *id != self_id && + self.filter.as_ref().map_or(true, |f| f.connection_allowed(&self_id, &id, ConnectionDirection::Outbound)) + ).take(min(max_handshakes_per_round, max_handshakes - handshake_count)) { self.connect_peer(&id, io); started += 1; } @@ -827,7 +834,7 @@ impl Host { Ok(SessionData::Ready) => { self.num_sessions.fetch_add(1, AtomicOrdering::SeqCst); let session_count = self.session_count(); - let (min_peers, max_peers, reserved_only) = { + let (min_peers, max_peers, reserved_only, self_id) = { let info = self.info.read(); let mut max_peers = info.config.max_peers; for cap in s.info.capabilities.iter() { @@ -836,7 +843,7 @@ impl Host { break; } } - (info.config.min_peers as usize, max_peers as usize, info.config.non_reserved_mode == NonReservedPeerMode::Deny) + (info.config.min_peers as usize, max_peers as usize, info.config.non_reserved_mode == NonReservedPeerMode::Deny, info.id().clone()) }; let id = s.id().expect("Ready session always has id").clone(); @@ -852,6 +859,14 @@ impl Host { break; } } + + if !self.filter.as_ref().map_or(true, |f| f.connection_allowed(&self_id, &id, ConnectionDirection::Inbound)) { + trace!(target: "network", "Inbound connection not allowed for {:?}", id); + s.disconnect(io, DisconnectReason::UnexpectedIdentity); + kill = true; + break; + } + ready_id = Some(id); // Add it to the node table @@ -1266,7 +1281,7 @@ fn host_client_url() { let mut config = NetworkConfiguration::new_local(); let key = "6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2".parse().unwrap(); config.use_secret = Some(key); - let host: Host = Host::new(config, Arc::new(NetworkStats::new())).unwrap(); + let host: Host = Host::new(config, Arc::new(NetworkStats::new()), None).unwrap(); assert!(host.local_url().starts_with("enode://101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c@")); } diff --git a/util/network/src/lib.rs b/util/network/src/lib.rs index 74e30a750..5695b8196 100644 --- a/util/network/src/lib.rs +++ b/util/network/src/lib.rs @@ -44,7 +44,7 @@ //! } //! //! fn main () { -//! let mut service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service"); +//! let mut service = NetworkService::new(NetworkConfiguration::new_local(), None).expect("Error creating network service"); //! service.start().expect("Error starting service"); //! service.register_protocol(Arc::new(MyHandler), *b"myp", 1, &[1u8]); //! @@ -95,6 +95,7 @@ mod error; mod node_table; mod stats; mod ip_utils; +mod connection_filter; #[cfg(test)] mod tests; @@ -104,6 +105,7 @@ pub use service::NetworkService; pub use error::NetworkError; pub use stats::NetworkStats; pub use session::SessionInfo; +pub use connection_filter::{ConnectionFilter, ConnectionDirection}; pub use io::TimerToken; pub use node_table::{is_valid_node_url, NodeId}; diff --git a/util/network/src/service.rs b/util/network/src/service.rs index d31edadb5..bce31e00a 100644 --- a/util/network/src/service.rs +++ b/util/network/src/service.rs @@ -22,6 +22,7 @@ use io::*; use parking_lot::RwLock; use std::sync::Arc; use ansi_term::Colour; +use connection_filter::ConnectionFilter; struct HostHandler { public_url: RwLock> @@ -48,11 +49,12 @@ pub struct NetworkService { stats: Arc, host_handler: Arc, config: NetworkConfiguration, + filter: Option>, } impl NetworkService { /// Starts IO event loop - pub fn new(config: NetworkConfiguration) -> Result { + pub fn new(config: NetworkConfiguration, filter: Option>) -> Result { let host_handler = Arc::new(HostHandler { public_url: RwLock::new(None) }); let io_service = IoService::::start()?; @@ -65,6 +67,7 @@ impl NetworkService { host: RwLock::new(None), config: config, host_handler: host_handler, + filter: filter, }) } @@ -115,7 +118,7 @@ impl NetworkService { pub fn start(&self) -> Result<(), NetworkError> { let mut host = self.host.write(); if host.is_none() { - let h = Arc::new(Host::new(self.config.clone(), self.stats.clone())?); + let h = Arc::new(Host::new(self.config.clone(), self.stats.clone(), self.filter.clone())?); self.io_service.register_handler(h.clone())?; *host = Some(h); } diff --git a/util/network/src/tests.rs b/util/network/src/tests.rs index 81325f57b..d743318ab 100644 --- a/util/network/src/tests.rs +++ b/util/network/src/tests.rs @@ -92,7 +92,7 @@ impl NetworkProtocolHandler for TestProtocol { #[test] fn net_service() { - let service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service"); + let service = NetworkService::new(NetworkConfiguration::new_local(), None).expect("Error creating network service"); service.start().unwrap(); service.register_protocol(Arc::new(TestProtocol::new(false)), *b"myp", 1, &[1u8]).unwrap(); } @@ -104,13 +104,13 @@ fn net_connect() { let mut config1 = NetworkConfiguration::new_local(); config1.use_secret = Some(key1.secret().clone()); config1.boot_nodes = vec![ ]; - let mut service1 = NetworkService::new(config1).unwrap(); + let mut service1 = NetworkService::new(config1, None).unwrap(); service1.start().unwrap(); let handler1 = TestProtocol::register(&mut service1, false); let mut config2 = NetworkConfiguration::new_local(); info!("net_connect: local URL: {}", service1.local_url().unwrap()); config2.boot_nodes = vec![ service1.local_url().unwrap() ]; - let mut service2 = NetworkService::new(config2).unwrap(); + let mut service2 = NetworkService::new(config2, None).unwrap(); service2.start().unwrap(); let handler2 = TestProtocol::register(&mut service2, false); while !handler1.got_packet() && !handler2.got_packet() && (service1.stats().sessions() == 0 || service2.stats().sessions() == 0) { @@ -123,7 +123,7 @@ fn net_connect() { #[test] fn net_start_stop() { let config = NetworkConfiguration::new_local(); - let service = NetworkService::new(config).unwrap(); + let service = NetworkService::new(config, None).unwrap(); service.start().unwrap(); service.stop().unwrap(); service.start().unwrap(); @@ -135,12 +135,12 @@ fn net_disconnect() { let mut config1 = NetworkConfiguration::new_local(); config1.use_secret = Some(key1.secret().clone()); config1.boot_nodes = vec![ ]; - let mut service1 = NetworkService::new(config1).unwrap(); + let mut service1 = NetworkService::new(config1, None).unwrap(); service1.start().unwrap(); let handler1 = TestProtocol::register(&mut service1, false); let mut config2 = NetworkConfiguration::new_local(); config2.boot_nodes = vec![ service1.local_url().unwrap() ]; - let mut service2 = NetworkService::new(config2).unwrap(); + let mut service2 = NetworkService::new(config2, None).unwrap(); service2.start().unwrap(); let handler2 = TestProtocol::register(&mut service2, true); while !(handler1.got_disconnect() && handler2.got_disconnect()) { @@ -153,7 +153,7 @@ fn net_disconnect() { #[test] fn net_timeout() { let config = NetworkConfiguration::new_local(); - let mut service = NetworkService::new(config).unwrap(); + let mut service = NetworkService::new(config, None).unwrap(); service.start().unwrap(); let handler = TestProtocol::register(&mut service, false); while !handler.got_timeout() { diff --git a/util/plain_hasher/Cargo.toml b/util/plain_hasher/Cargo.toml new file mode 100644 index 000000000..0ed45ba36 --- /dev/null +++ b/util/plain_hasher/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "plain_hasher" +description = "Hasher for 32-bit keys." +version = "0.1.0" +authors = ["Parity Technologies "] +license = "MIT" +keywords = ["hash", "hasher"] +categories = ["no-std"] +homepage = "https://github.com/paritytech/plain_hasher" + +[dependencies] +crunchy = "0.1.6" diff --git a/util/plain_hasher/benches/bench.rs b/util/plain_hasher/benches/bench.rs new file mode 100644 index 000000000..e7e8570ab --- /dev/null +++ b/util/plain_hasher/benches/bench.rs @@ -0,0 +1,33 @@ +#![feature(test)] + +extern crate test; +extern crate plain_hasher; + +use std::hash::Hasher; +use std::collections::hash_map::DefaultHasher; +use test::{Bencher, black_box}; +use plain_hasher::PlainHasher; + +#[bench] +fn write_plain_hasher(b: &mut Bencher) { + b.iter(|| { + let n: u8 = black_box(100); + (0..n).fold(PlainHasher::default(), |mut old, new| { + let bb = black_box([new; 32]); + old.write(&bb as &[u8]); + old + }); + }); +} + +#[bench] +fn write_default_hasher(b: &mut Bencher) { + b.iter(|| { + let n: u8 = black_box(100); + (0..n).fold(DefaultHasher::default(), |mut old, new| { + let bb = black_box([new; 32]); + old.write(&bb as &[u8]); + old + }); + }); +} diff --git a/util/plain_hasher/src/lib.rs b/util/plain_hasher/src/lib.rs new file mode 100644 index 000000000..b16596363 --- /dev/null +++ b/util/plain_hasher/src/lib.rs @@ -0,0 +1,55 @@ +#![no_std] +#[macro_use] +extern crate crunchy; + +use core::{hash, mem}; + +/// Hasher that just takes 8 bytes of the provided value. +/// May only be used for keys which are 32 bytes. +#[derive(Default)] +pub struct PlainHasher { + prefix: u64, +} + +impl hash::Hasher for PlainHasher { + #[inline] + fn finish(&self) -> u64 { + self.prefix + } + + #[inline] + #[allow(unused_assignments)] + fn write(&mut self, bytes: &[u8]) { + debug_assert!(bytes.len() == 32); + + unsafe { + let mut bytes_ptr = bytes.as_ptr(); + let prefix_u8: &mut [u8; 8] = mem::transmute(&mut self.prefix); + let mut prefix_ptr = prefix_u8.as_mut_ptr(); + + unroll! { + for _i in 0..8 { + *prefix_ptr ^= (*bytes_ptr ^ *bytes_ptr.offset(8)) ^ (*bytes_ptr.offset(16) ^ *bytes_ptr.offset(24)); + + bytes_ptr = bytes_ptr.offset(1); + prefix_ptr = prefix_ptr.offset(1); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use core::hash::Hasher; + use super::PlainHasher; + + #[test] + fn it_works() { + let mut bytes = [32u8; 32]; + bytes[0] = 15; + let mut hasher = PlainHasher::default(); + hasher.write(&bytes); + assert_eq!(hasher.prefix, 47); + } +} diff --git a/util/reactor/src/lib.rs b/util/reactor/src/lib.rs index e5f04d652..f5a37fb0f 100644 --- a/util/reactor/src/lib.rs +++ b/util/reactor/src/lib.rs @@ -20,7 +20,7 @@ extern crate futures; extern crate tokio_core; -use std::thread; +use std::{fmt, thread}; use std::sync::mpsc; use std::time::Duration; use futures::{Future, IntoFuture}; @@ -81,7 +81,19 @@ enum Mode { ThreadPerFuture, } -#[derive(Clone)] +impl fmt::Debug for Mode { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + use self::Mode::*; + + match *self { + Tokio(_) => write!(fmt, "tokio"), + Sync => write!(fmt, "synchronous"), + ThreadPerFuture => write!(fmt, "thread per future"), + } + } +} + +#[derive(Debug, Clone)] pub struct Remote { inner: Mode, } diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index fc893654a..8c87ef623 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -17,6 +17,7 @@ //! Disk-backed `HashDB` implementation. use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use rlp::*; use hashdb::*; @@ -66,23 +67,28 @@ impl ArchiveDB { impl HashDB for ArchiveDB { fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter(self.column) { - let h = H256::from_slice(&*key); - ret.insert(h, 1); - } + let mut ret: HashMap = self.backing.iter(self.column) + .map(|(key, _)| (H256::from_slice(&*key), 1)) + .collect(); for (key, refs) in self.overlay.keys() { - let refs = *ret.get(&key).unwrap_or(&0) + refs; - ret.insert(key, refs); + match ret.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() += refs; + }, + Entry::Vacant(entry) => { + entry.insert(refs); + } + } } ret } fn get(&self, key: &H256) -> Option { - let k = self.overlay.raw(key); - if let Some((d, rc)) = k { - if rc > 0 { return Some(d); } + if let Some((d, rc)) = self.overlay.raw(key) { + if rc > 0 { + return Some(d); + } } self.payload(key) } diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index b7b57d537..0d0681a03 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -18,6 +18,7 @@ use std::fmt; use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use parking_lot::RwLock; use heapsize::HeapSizeOf; @@ -159,33 +160,38 @@ impl EarlyMergeDB { fn insert_keys(inserts: &[(H256, DBValue)], backing: &KeyValueDB, col: Option, refs: &mut HashMap, batch: &mut DBTransaction, trace: bool) { for &(ref h, ref d) in inserts { - if let Some(c) = refs.get_mut(h) { - // already counting. increment. - c.queue_refs += 1; - if trace { - trace!(target: "jdb.fine", " insert({}): In queue: Incrementing refs to {}", h, c.queue_refs); - } - continue; - } - - // this is the first entry for this node in the journal. - if backing.get(col, h).expect("Low-level database error. Some issue with your hard disk?").is_some() { - // already in the backing DB. start counting, and remember it was already in. - Self::set_already_in(batch, col, h); - refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: true}); - if trace { - trace!(target: "jdb.fine", " insert({}): New to queue, in DB: Recording and inserting into queue", h); - } - continue; - } - - // Gets removed when a key leaves the journal, so should never be set when we're placing a new key. - //Self::reset_already_in(&h); - assert!(!Self::is_already_in(backing, col, &h)); - batch.put(col, h, d); - refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: false}); - if trace { - trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h); + match refs.entry(*h) { + Entry::Occupied(mut entry) => { + let info = entry.get_mut(); + // already counting. increment. + info.queue_refs += 1; + if trace { + trace!(target: "jdb.fine", " insert({}): In queue: Incrementing refs to {}", h, info.queue_refs); + } + }, + Entry::Vacant(entry) => { + // this is the first entry for this node in the journal. + let in_archive = backing.get(col, h).expect("Low-level database error. Some issue with your hard disk?").is_some(); + if in_archive { + // already in the backing DB. start counting, and remember it was already in. + Self::set_already_in(batch, col, h); + if trace { + trace!(target: "jdb.fine", " insert({}): New to queue, in DB: Recording and inserting into queue", h); + } + } else { + // Gets removed when a key leaves the journal, so should never be set when we're placing a new key. + //Self::reset_already_in(&h); + assert!(!Self::is_already_in(backing, col, h)); + if trace { + trace!(target: "jdb.fine", " insert({}): New to queue, not in DB: Inserting into queue and DB", h); + } + batch.put(col, h, d); + } + entry.insert(RefInfo { + queue_refs: 1, + in_archive: in_archive, + }); + }, } } } @@ -193,15 +199,20 @@ impl EarlyMergeDB { fn replay_keys(inserts: &[H256], backing: &KeyValueDB, col: Option, refs: &mut HashMap) { trace!(target: "jdb.fine", "replay_keys: inserts={:?}, refs={:?}", inserts, refs); for h in inserts { - if let Some(c) = refs.get_mut(h) { + match refs.entry(*h) { // already counting. increment. - c.queue_refs += 1; - continue; + Entry::Occupied(mut entry) => { + entry.get_mut().queue_refs += 1; + }, + // this is the first entry for this node in the journal. + // it is initialised to 1 if it was already in. + Entry::Vacant(entry) => { + entry.insert(RefInfo { + queue_refs: 1, + in_archive: Self::is_already_in(backing, col, h), + }); + }, } - - // this is the first entry for this node in the journal. - // it is initialised to 1 if it was already in. - refs.insert(h.clone(), RefInfo{queue_refs: 1, in_archive: Self::is_already_in(backing, col, h)}); } trace!(target: "jdb.fine", "replay_keys: (end) refs={:?}", refs); } @@ -213,50 +224,54 @@ impl EarlyMergeDB { // (the latter option would then mean removing the RefInfo, since it would no longer be counted in the queue.) // both are valid, but we switch between them depending on context. // All inserts in queue (i.e. those which may yet be reverted) have an entry in refs. - for h in deletes.iter() { - let mut n: Option = None; - if let Some(c) = refs.get_mut(h) { - if c.in_archive && from == RemoveFrom::Archive { - c.in_archive = false; - Self::reset_already_in(batch, col, h); - if trace { - trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Reducing to queue only and recording", h); + for h in deletes { + match refs.entry(*h) { + Entry::Occupied(mut entry) => { + if entry.get().in_archive && from == RemoveFrom::Archive { + entry.get_mut().in_archive = false; + Self::reset_already_in(batch, col, h); + if trace { + trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Reducing to queue only and recording", h); + } + continue; } - continue; - } else if c.queue_refs > 1 { - c.queue_refs -= 1; - if trace { - trace!(target: "jdb.fine", " remove({}): In queue > 1 refs: Decrementing ref count to {}", h, c.queue_refs); + if entry.get().queue_refs > 1 { + entry.get_mut().queue_refs -= 1; + if trace { + trace!(target: "jdb.fine", " remove({}): In queue > 1 refs: Decrementing ref count to {}", h, entry.get().queue_refs); + } + continue; } - continue; - } else { - n = Some(c.clone()); - } - } - match n { - Some(RefInfo{queue_refs: 1, in_archive: true}) => { - refs.remove(h); - Self::reset_already_in(batch, col, h); - if trace { - trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Removing from queue and leaving in archive", h); + + let queue_refs = entry.get().queue_refs; + let in_archive = entry.get().in_archive; + + match (queue_refs, in_archive) { + (1, true) => { + entry.remove(); + Self::reset_already_in(batch, col, h); + if trace { + trace!(target: "jdb.fine", " remove({}): In archive, 1 in queue: Removing from queue and leaving in archive", h); + } + }, + (1, false) => { + entry.remove(); + batch.delete(col, h); + if trace { + trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h); + } + }, + _ => panic!("Invalid value in refs: {:?}", entry.get()), } - } - Some(RefInfo{queue_refs: 1, in_archive: false}) => { - refs.remove(h); - batch.delete(col, h); - if trace { - trace!(target: "jdb.fine", " remove({}): Not in archive, only 1 ref in queue: Removing from queue and DB", h); - } - } - None => { + }, + Entry::Vacant(_entry) => { // Gets removed when moving from 1 to 0 additional refs. Should never be here at 0 additional refs. //assert!(!Self::is_already_in(db, &h)); batch.delete(col, h); if trace { trace!(target: "jdb.fine", " remove({}): Not in queue - MUST BE IN ARCHIVE: Removing from DB", h); } - } - _ => panic!("Invalid value in refs: {:?}", n), + }, } } } @@ -311,23 +326,28 @@ impl EarlyMergeDB { impl HashDB for EarlyMergeDB { fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter(self.column) { - let h = H256::from_slice(&*key); - ret.insert(h, 1); - } + let mut ret: HashMap = self.backing.iter(self.column) + .map(|(key, _)| (H256::from_slice(&*key), 1)) + .collect(); for (key, refs) in self.overlay.keys() { - let refs = *ret.get(&key).unwrap_or(&0) + refs; - ret.insert(key, refs); + match ret.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() += refs; + }, + Entry::Vacant(entry) => { + entry.insert(refs); + } + } } ret } fn get(&self, key: &H256) -> Option { - let k = self.overlay.raw(key); - if let Some((d, rc)) = k { - if rc > 0 { return Some(d) } + if let Some((d, rc)) = self.overlay.raw(key) { + if rc > 0 { + return Some(d) + } } self.payload(key) } diff --git a/util/src/journaldb/overlayrecentdb.rs b/util/src/journaldb/overlayrecentdb.rs index e96430e06..975abd8e8 100644 --- a/util/src/journaldb/overlayrecentdb.rs +++ b/util/src/journaldb/overlayrecentdb.rs @@ -17,6 +17,7 @@ //! `JournalDB` over in-memory overlay use std::collections::HashMap; +use std::collections::hash_map::Entry; use std::sync::Arc; use parking_lot::RwLock; use heapsize::HeapSizeOf; @@ -407,23 +408,28 @@ impl JournalDB for OverlayRecentDB { impl HashDB for OverlayRecentDB { fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter(self.column) { - let h = H256::from_slice(&*key); - ret.insert(h, 1); - } + let mut ret: HashMap = self.backing.iter(self.column) + .map(|(key, _)| (H256::from_slice(&*key), 1)) + .collect(); for (key, refs) in self.transaction_overlay.keys() { - let refs = *ret.get(&key).unwrap_or(&0) + refs; - ret.insert(key, refs); + match ret.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() += refs; + }, + Entry::Vacant(entry) => { + entry.insert(refs); + } + } } ret } fn get(&self, key: &H256) -> Option { - let k = self.transaction_overlay.raw(key); - if let Some((d, rc)) = k { - if rc > 0 { return Some(d) } + if let Some((d, rc)) = self.transaction_overlay.raw(key) { + if rc > 0 { + return Some(d) + } } let v = { let journal_overlay = self.journal_overlay.read(); diff --git a/util/src/journaldb/refcounteddb.rs b/util/src/journaldb/refcounteddb.rs index e38994700..1b9c9ded8 100644 --- a/util/src/journaldb/refcounteddb.rs +++ b/util/src/journaldb/refcounteddb.rs @@ -198,7 +198,7 @@ impl JournalDB for RefCountedDB { fn consolidate(&mut self, mut with: MemoryDB) { for (key, (value, rc)) in with.drain() { for _ in 0..rc { - self.emplace(key.clone(), value.clone()); + self.emplace(key, value.clone()); } for _ in rc..0 { diff --git a/util/src/memorydb.rs b/util/src/memorydb.rs index cc8d0a3de..6c9cb63fe 100644 --- a/util/src/memorydb.rs +++ b/util/src/memorydb.rs @@ -16,14 +16,14 @@ //! Reference-counted memory-based `HashDB` implementation. -use hash::*; -use rlp::*; -use sha3::*; -use hashdb::*; -use heapsize::*; use std::mem; use std::collections::HashMap; use std::collections::hash_map::Entry; +use heapsize::HeapSizeOf; +use hash::{H256FastMap, H256}; +use rlp::NULL_RLP; +use sha3::*; +use hashdb::*; /// Reference-counted memory-based `HashDB` implementation. /// @@ -181,7 +181,13 @@ impl HashDB for MemoryDB { } fn keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect() + self.data.iter() + .filter_map(|(k, v)| if v.1 != 0 { + Some((*k, v.1)) + } else { + None + }) + .collect() } fn contains(&self, key: &H256) -> bool { @@ -200,16 +206,17 @@ impl HashDB for MemoryDB { return SHA3_NULL_RLP.clone(); } let key = value.sha3(); - if match self.data.get_mut(&key) { - Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { - *old_value = DBValue::from_slice(value); + match self.data.entry(key) { + Entry::Occupied(mut entry) => { + let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); + if *rc >= -0x80000000i32 && *rc <= 0 { + *old_value = DBValue::from_slice(value); + } *rc += 1; - false }, - Some(&mut (_, ref mut x)) => { *x += 1; false } , - None => true, - }{ // ... None falls through into... - self.data.insert(key.clone(), (DBValue::from_slice(value), 1)); + Entry::Vacant(entry) => { + entry.insert((DBValue::from_slice(value), 1)); + }, } key } @@ -219,17 +226,18 @@ impl HashDB for MemoryDB { return; } - match self.data.get_mut(&key) { - Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { - *old_value = value; + match self.data.entry(key) { + Entry::Occupied(mut entry) => { + let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); + if *rc >= -0x80000000i32 && *rc <= 0 { + *old_value = value; + } *rc += 1; - return; }, - Some(&mut (_, ref mut x)) => { *x += 1; return; } , - None => {}, + Entry::Vacant(entry) => { + entry.insert((value, 1)); + }, } - // ... None falls through into... - self.data.insert(key, (value, 1)); } fn remove(&mut self, key: &H256) { @@ -237,11 +245,14 @@ impl HashDB for MemoryDB { return; } - if match self.data.get_mut(key) { - Some(&mut (_, ref mut x)) => { *x -= 1; false } - None => true - }{ // ... None falls through into... - self.data.insert(key.clone(), (DBValue::new(), -1)); + match self.data.entry(*key) { + Entry::Occupied(mut entry) => { + let &mut (_, ref mut rc) = entry.get_mut(); + *rc -= 1; + }, + Entry::Vacant(entry) => { + entry.insert((DBValue::new(), -1)); + }, } } } diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index 6b6f501a7..80cc499b9 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -16,13 +16,14 @@ //! Disk-backed `HashDB` implementation. +use std::sync::Arc; +use std::collections::HashMap; +use std::collections::hash_map::Entry; use error::*; use hash::*; use rlp::*; use hashdb::*; use memorydb::*; -use std::sync::*; -use std::collections::HashMap; use kvdb::{KeyValueDB, DBTransaction}; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay. @@ -125,19 +126,27 @@ impl OverlayDB { impl HashDB for OverlayDB { fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iter(self.column) { - let h = H256::from_slice(&*key); - let r = self.payload(&h).unwrap().1; - ret.insert(h, r as i32); - } + let mut ret: HashMap = self.backing.iter(self.column) + .map(|(key, _)| { + let h = H256::from_slice(&*key); + let r = self.payload(&h).unwrap().1; + (h, r as i32) + }) + .collect(); for (key, refs) in self.overlay.keys() { - let refs = *ret.get(&key).unwrap_or(&0) + refs; - ret.insert(key, refs); + match ret.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() += refs; + }, + Entry::Vacant(entry) => { + entry.insert(refs); + } + } } ret } + fn get(&self, key: &H256) -> Option { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. @@ -165,6 +174,7 @@ impl HashDB for OverlayDB { _ => None, } } + fn contains(&self, key: &H256) -> bool { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. @@ -185,6 +195,7 @@ impl HashDB for OverlayDB { } } } + fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } fn emplace(&mut self, key: H256, value: DBValue) { self.overlay.emplace(key, value); } fn remove(&mut self, key: &H256) { self.overlay.remove(key); } diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index e9f4bc671..3e62550ba 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -217,59 +217,66 @@ impl<'a> TrieDBIterator<'a> { Ok(r) } - fn seek_descend<'key>(&mut self, node_data: DBValue, key: &NibbleSlice<'key>) -> super::Result<()> { - let node = Node::decoded(&node_data); - match node { - Node::Leaf(ref slice, _) => { - if slice == key { - self.trail.push(Crumb { - status: Status::At, - node: node.clone().into(), - }); - } else { - self.trail.push(Crumb { - status: Status::Exiting, - node: node.clone().into(), - }); - } + fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<()> { + loop { + let (data, mid) = { + let node = Node::decoded(&node_data); + match node { + Node::Leaf(slice, _) => { + if slice == key { + self.trail.push(Crumb { + status: Status::At, + node: node.clone().into(), + }); + } else { + self.trail.push(Crumb { + status: Status::Exiting, + node: node.clone().into(), + }); + } - self.key_nibbles.extend(slice.iter()); - Ok(()) - }, - Node::Extension(ref slice, ref item) => { - if key.starts_with(slice) { - self.trail.push(Crumb { - status: Status::At, - node: node.clone().into(), - }); - self.key_nibbles.extend(slice.iter()); - let data = self.db.get_raw_or_lookup(&*item)?; - self.seek_descend(data, &key.mid(slice.len())) - } else { - self.descend(&node_data)?; - Ok(()) + self.key_nibbles.extend(slice.iter()); + return Ok(()) + }, + Node::Extension(ref slice, ref item) => { + if key.starts_with(slice) { + self.trail.push(Crumb { + status: Status::At, + node: node.clone().into(), + }); + self.key_nibbles.extend(slice.iter()); + let data = self.db.get_raw_or_lookup(&*item)?; + (data, slice.len()) + } else { + self.descend(&node_data)?; + return Ok(()) + } + }, + Node::Branch(ref nodes, _) => match key.is_empty() { + true => { + self.trail.push(Crumb { + status: Status::At, + node: node.clone().into(), + }); + return Ok(()) + }, + false => { + let i = key.at(0); + self.trail.push(Crumb { + status: Status::AtChild(i as usize), + node: node.clone().into(), + }); + self.key_nibbles.push(i); + let child = self.db.get_raw_or_lookup(&*nodes[i as usize])?; + (child, 1) + } + }, + _ => return Ok(()), } - }, - Node::Branch(ref nodes, _) => match key.is_empty() { - true => { - self.trail.push(Crumb { - status: Status::At, - node: node.clone().into(), - }); - Ok(()) - }, - false => { - let i = key.at(0); - self.trail.push(Crumb { - status: Status::AtChild(i as usize), - node: node.clone().into(), - }); - self.key_nibbles.push(i); - let child = self.db.get_raw_or_lookup(&*nodes[i as usize])?; - self.seek_descend(child, &key.mid(1)) - } - }, - _ => Ok(()) + }; + + node_data = data; + key = key.mid(mid); } } @@ -314,7 +321,7 @@ impl<'a> TrieIterator for TrieDBIterator<'a> { self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; - self.seek_descend(root_rlp, &NibbleSlice::new(key)) + self.seek(root_rlp, NibbleSlice::new(key)) } } diff --git a/util/src/triehash.rs b/util/src/triehash.rs index ca1974e7b..62d3dbf97 100644 --- a/util/src/triehash.rs +++ b/util/src/triehash.rs @@ -115,7 +115,7 @@ pub fn sec_trie_root(input: Vec<(Vec, Vec)>) -> H256 { let gen_input = input // first put elements into btree to sort them and to remove duplicates .into_iter() - .map(|(k, v)| (k.sha3().to_vec(), v)) + .map(|(k, v)| (k.sha3(), v)) .collect::>() // then move them to a vector .into_iter() @@ -155,8 +155,7 @@ fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { let oddness_factor = inlen % 2; // next even number divided by two let reslen = (inlen + 2) >> 1; - let mut res = vec![]; - res.reserve(reslen); + let mut res = Vec::with_capacity(reslen); let first_byte = { let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; @@ -180,11 +179,11 @@ fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { /// Converts slice of bytes to nibbles. fn as_nibbles(bytes: &[u8]) -> Vec { - let mut res = vec![]; - res.reserve(bytes.len() * 2); + let mut res = Vec::with_capacity(bytes.len() * 2); for i in 0..bytes.len() { - res.push(bytes[i] >> 4); - res.push((bytes[i] << 4) >> 4); + let byte = bytes[i]; + res.push(byte >> 4); + res.push(byte & 0b1111); } res }