From 2789824a51f06eacbf833a7c55ef1b32a8f55669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 31 Aug 2016 16:53:22 +0200 Subject: [PATCH 1/8] Much nicer error pages --- dapps/src/apps/fetcher.rs | 10 ++++----- dapps/src/error_tpl.html | 22 +++++++++++++++++++ dapps/src/handlers/content.rs | 33 ++++++++++++++++++++-------- dapps/src/handlers/fetch.rs | 32 +++++++++++++++++---------- dapps/src/router/auth.rs | 5 ++--- dapps/src/router/host_validation.rs | 12 +++++----- signer/src/ws_server/session.rs | 34 +++++++++++++++++++++++++---- 7 files changed, 108 insertions(+), 40 deletions(-) create mode 100644 dapps/src/error_tpl.html diff --git a/dapps/src/apps/fetcher.rs b/dapps/src/apps/fetcher.rs index e31aae55d..6223fb5e8 100644 --- a/dapps/src/apps/fetcher.rs +++ b/dapps/src/apps/fetcher.rs @@ -98,13 +98,11 @@ impl AppFetcher { }, // App is already being fetched Some(&mut ContentStatus::Fetching(_)) => { - (None, Box::new(ContentHandler::html( + (None, Box::new(ContentHandler::error_with_refresh( StatusCode::ServiceUnavailable, - format!( - "{}{}", - "", - "

This dapp is already being downloaded.

Please wait...

", - ) + "Download In Progress", + "This dapp is already being downloaded. Please wait...", + None, )) as Box) }, // We need to start fetching app diff --git a/dapps/src/error_tpl.html b/dapps/src/error_tpl.html new file mode 100644 index 000000000..227764b9c --- /dev/null +++ b/dapps/src/error_tpl.html @@ -0,0 +1,22 @@ + + + + + + {meta} + {title} + + + +
+
+
+

{title}

+

{message}

+

{details}

+
+
+ {version} +
+ + diff --git a/dapps/src/handlers/content.rs b/dapps/src/handlers/content.rs index 092c417ac..f283fbb6a 100644 --- a/dapps/src/handlers/content.rs +++ b/dapps/src/handlers/content.rs @@ -21,6 +21,8 @@ use hyper::{header, server, Decoder, Encoder, Next}; use hyper::net::HttpStream; use hyper::status::StatusCode; +use util::version; + pub struct ContentHandler { code: StatusCode, content: String, @@ -38,15 +40,6 @@ impl ContentHandler { } } - pub fn forbidden(content: String, mimetype: String) -> Self { - ContentHandler { - code: StatusCode::Forbidden, - content: content, - mimetype: mimetype, - write_pos: 0 - } - } - pub fn not_found(content: String, mimetype: String) -> Self { ContentHandler { code: StatusCode::NotFound, @@ -60,6 +53,28 @@ impl ContentHandler { Self::new(code, content, "text/html".into()) } + pub fn error(code: StatusCode, title: &str, message: &str, details: Option<&str>) -> Self { + Self::html(code, format!( + include_str!("../error_tpl.html"), + title=title, + meta="", + message=message, + details=details.unwrap_or_else(|| ""), + version=version(), + )) + } + + pub fn error_with_refresh(code: StatusCode, title: &str, message: &str, details: Option<&str>) -> Self { + Self::html(code, format!( + include_str!("../error_tpl.html"), + title=title, + meta="", + message=message, + details=details.unwrap_or_else(|| ""), + version=version(), + )) + } + pub fn new(code: StatusCode, content: String, mimetype: String) -> Self { ContentHandler { code: code, diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index d4919562a..7cbb2537b 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -130,16 +130,20 @@ impl server::Handler for ContentFetcherHandler< deadline: Instant::now() + Duration::from_secs(FETCH_TIMEOUT), receiver: receiver, }, - Err(e) => FetchState::Error(ContentHandler::html( + Err(e) => FetchState::Error(ContentHandler::error( StatusCode::BadGateway, - format!("

Error starting dapp download.

{}
", e), + "Unable To Start Dapp Download", + "Could not initialize download of the dapp. It might be a problem with remote server.", + Some(&format!("{}", e)), )), } }, // or return error - _ => FetchState::Error(ContentHandler::html( + _ => FetchState::Error(ContentHandler::error( StatusCode::MethodNotAllowed, - "

Only GET requests are allowed.

".into(), + "Method Not Allowed", + "Only GET requests are allowed.", + None, )), }) } else { None }; @@ -156,10 +160,12 @@ impl server::Handler for ContentFetcherHandler< // Request may time out FetchState::InProgress { ref deadline, .. } if *deadline < Instant::now() => { trace!(target: "dapps", "Fetching dapp failed because of timeout."); - let timeout = ContentHandler::html( + let timeout = ContentHandler::error( StatusCode::GatewayTimeout, - format!("

Could not fetch app bundle within {} seconds.

", FETCH_TIMEOUT), - ); + "Download Timeout", + &format!("Could not fetch dapp bundle within {} seconds.", FETCH_TIMEOUT), + None + ); Self::close_client(&mut self.client); (Some(FetchState::Error(timeout)), Next::write()) }, @@ -175,9 +181,11 @@ impl server::Handler for ContentFetcherHandler< let state = match self.dapp.validate_and_install(path.clone()) { Err(e) => { trace!(target: "dapps", "Error while validating dapp: {:?}", e); - FetchState::Error(ContentHandler::html( + FetchState::Error(ContentHandler::error( StatusCode::BadGateway, - format!("

Downloaded bundle does not contain valid app.

{:?}
", e), + "Invalid Dapp", + "Downloaded bundle does not contain a valid dapp.", + Some(&format!("{:?}", e)) )) }, Ok(manifest) => FetchState::Done(manifest) @@ -188,9 +196,11 @@ impl server::Handler for ContentFetcherHandler< }, Ok(Err(e)) => { warn!(target: "dapps", "Unable to fetch new dapp: {:?}", e); - let error = ContentHandler::html( + let error = ContentHandler::error( StatusCode::BadGateway, - "

There was an error when fetching the dapp.

".into(), + "Download Error", + "There was an error when fetching the dapp.", + Some(&format!("{:?}", e)), ); (Some(FetchState::Error(error)), Next::write()) }, diff --git a/dapps/src/router/auth.rs b/dapps/src/router/auth.rs index d18424a00..596796eed 100644 --- a/dapps/src/router/auth.rs +++ b/dapps/src/router/auth.rs @@ -55,10 +55,9 @@ impl Authorization for HttpBasicAuth { match auth { Access::Denied => { - Authorized::No(Box::new(ContentHandler::new( + Authorized::No(Box::new(ContentHandler::error( status::StatusCode::Unauthorized, - "

Unauthorized

".into(), - "text/html".into(), + "Unauthorized", "You need to provide valid credentials to access this page.", None ))) }, Access::AuthRequired => { diff --git a/dapps/src/router/host_validation.rs b/dapps/src/router/host_validation.rs index e0f974482..d1d651c5d 100644 --- a/dapps/src/router/host_validation.rs +++ b/dapps/src/router/host_validation.rs @@ -16,7 +16,7 @@ use DAPPS_DOMAIN; -use hyper::{server, header}; +use hyper::{server, header, StatusCode}; use hyper::net::HttpStream; use jsonrpc_http_server::{is_host_header_valid}; @@ -38,11 +38,9 @@ pub fn is_valid(request: &server::Request, allowed_hosts: &[String], } pub fn host_invalid_response() -> Box + Send> { - Box::new(ContentHandler::forbidden( - r#" -

Request with disallowed Host header has been blocked.

-

Check the URL in your browser address bar.

- "#.into(), - "text/html".into() + Box::new(ContentHandler::error(StatusCode::Forbidden, + "Current host is disallowed", + "You are trying to access your node using incorrect address.", + Some("Use allowed URL or specify different hosts CLI options.") )) } diff --git a/signer/src/ws_server/session.rs b/signer/src/ws_server/session.rs index e7abea833..34097c24d 100644 --- a/signer/src/ws_server/session.rs +++ b/signer/src/ws_server/session.rs @@ -22,7 +22,7 @@ use std::path::{PathBuf, Path}; use std::sync::Arc; use std::str::FromStr; use jsonrpc_core::IoHandler; -use util::H256; +use util::{H256, version}; #[cfg(feature = "ui")] mod signer { @@ -112,7 +112,12 @@ impl ws::Handler for Session { if !self.skip_origin_validation { if !origin_is_allowed(&self.self_origin, origin) && !(origin.is_none() && origin_is_allowed(&self.self_origin, host)) { warn!(target: "signer", "Blocked connection to Signer API from untrusted origin."); - return Ok(ws::Response::forbidden(format!("You are not allowed to access system ui. Use: http://{}", self.self_origin))); + return Ok(error( + ErrorType::Forbidden, + "URL Blocked", + "You are not allowed to access Trusted Signer using this URL.", + Some(&format!("Use: http://{}", self.self_origin)), + )); } } @@ -121,7 +126,7 @@ impl ws::Handler for Session { // Check authorization if !auth_is_valid(&self.authcodes_path, req.protocols()) { info!(target: "signer", "Unauthorized connection to Signer API blocked."); - return Ok(ws::Response::forbidden("You are not authorized.".into())); + return Ok(error(ErrorType::Forbidden, "Not Authorized", "Request to this API was not authorized.", None)); } let protocols = req.protocols().expect("Existence checked by authorization."); @@ -137,7 +142,7 @@ impl ws::Handler for Session { Ok(signer::handle(req.resource()) .map_or_else( // return 404 not found - || add_headers(ws::Response::not_found("Not found".into()), "text/plain"), + || error(ErrorType::NotFound, "Not found", "Requested file was not found.", None), // or serve the file |f| add_headers(ws::Response::ok(f.content.into()), &f.mime) )) @@ -185,3 +190,24 @@ impl ws::Factory for Factory { } } } + +enum ErrorType { + NotFound, + Forbidden, +} + +fn error(error: ErrorType, title: &str, message: &str, details: Option<&str>) -> ws::Response { + let content = format!( + include_str!("../../../dapps/src/error_tpl.html"), + title=title, + meta="", + message=message, + details=details.unwrap_or(""), + version=version(), + ); + let res = match error { + ErrorType::NotFound => ws::Response::not_found(content), + ErrorType::Forbidden => ws::Response::forbidden(content), + }; + add_headers(res, "text/html") +} From 9c4d31f5489c7f7e45113c31f032c43f640df4b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 1 Sep 2016 10:16:04 +0200 Subject: [PATCH 2/8] Signer errors --- Cargo.lock | 12 +++-- dapps/Cargo.toml | 1 + dapps/src/error_tpl.html | 2 +- dapps/src/lib.rs | 2 + dapps/src/router/mod.rs | 13 +++-- dapps/src/tests/authorization.rs | 2 +- dapps/src/tests/helpers.rs | 50 ++---------------- dapps/src/tests/validation.rs | 23 +++++++- devtools/src/http_client.rs | 64 +++++++++++++++++++++++ devtools/src/lib.rs | 1 + signer/Cargo.toml | 1 + signer/src/lib.rs | 10 ++-- signer/src/tests/mod.rs | 81 +++++++++++++++++++++++++++++ signer/src/ws_server/error_tpl.html | 21 ++++++++ signer/src/ws_server/mod.rs | 9 +++- signer/src/ws_server/session.rs | 14 +++-- 16 files changed, 236 insertions(+), 70 deletions(-) create mode 100644 devtools/src/http_client.rs create mode 100644 signer/src/tests/mod.rs create mode 100644 signer/src/ws_server/error_tpl.html diff --git a/Cargo.lock b/Cargo.lock index 79bb6dba9..b9ad5d6bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,7 @@ version = "1.4.0" dependencies = [ "clippy 0.0.85 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore-devtools 1.4.0", "ethcore-rpc 1.4.0", "ethcore-util 1.4.0", "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", @@ -455,6 +456,7 @@ version = "1.4.0" dependencies = [ "clippy 0.0.85 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore-devtools 1.4.0", "ethcore-io 1.4.0", "ethcore-rpc 1.4.0", "ethcore-util 1.4.0", @@ -1068,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "parity-dapps" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a" +source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" dependencies = [ "aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1082,7 +1084,7 @@ dependencies = [ [[package]] name = "parity-dapps-home" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a" +source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1090,7 +1092,7 @@ dependencies = [ [[package]] name = "parity-dapps-signer" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a" +source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1098,7 +1100,7 @@ dependencies = [ [[package]] name = "parity-dapps-status" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a" +source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1106,7 +1108,7 @@ dependencies = [ [[package]] name = "parity-dapps-wallet" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a" +source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 1f5a0c491..f2604f07f 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -23,6 +23,7 @@ serde_macros = { version = "0.7.0", optional = true } zip = { version = "0.1", default-features = false } ethabi = "0.2.1" linked-hash-map = "0.3" +ethcore-devtools = { path = "../devtools" } ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } diff --git a/dapps/src/error_tpl.html b/dapps/src/error_tpl.html index 227764b9c..6551431a6 100644 --- a/dapps/src/error_tpl.html +++ b/dapps/src/error_tpl.html @@ -16,7 +16,7 @@

{details}

- {version} + {version}
diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index bc54b0f37..e24def0a1 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -61,6 +61,8 @@ extern crate parity_dapps; extern crate ethcore_rpc; extern crate ethcore_util as util; extern crate linked_hash_map; +#[cfg(test)] +extern crate ethcore_devtools as devtools; mod endpoint; mod apps; diff --git a/dapps/src/router/mod.rs b/dapps/src/router/mod.rs index 94d0a0fc0..9326917bf 100644 --- a/dapps/src/router/mod.rs +++ b/dapps/src/router/mod.rs @@ -55,9 +55,16 @@ pub struct Router { impl server::Handler for Router { fn on_request(&mut self, req: server::Request) -> Next { + + // Choose proper handler depending on path / domain + let url = extract_url(&req); + let endpoint = extract_endpoint(&url); + let is_utils = endpoint.1 == SpecialEndpoint::Utils; + // Validate Host header if let Some(ref hosts) = self.allowed_hosts { - if !host_validation::is_valid(&req, hosts, self.endpoints.keys().cloned().collect()) { + let is_valid = is_utils || host_validation::is_valid(&req, hosts, self.endpoints.keys().cloned().collect()); + if !is_valid { self.handler = host_validation::host_invalid_response(); return self.handler.on_request(req); } @@ -70,10 +77,6 @@ impl server::Handler for Router { return self.handler.on_request(req); } - // Choose proper handler depending on path / domain - let url = extract_url(&req); - let endpoint = extract_endpoint(&url); - self.handler = match endpoint { // First check special endpoints (ref path, ref endpoint) if self.special.contains_key(endpoint) => { diff --git a/dapps/src/tests/authorization.rs b/dapps/src/tests/authorization.rs index dceb194b7..e7a70c9cd 100644 --- a/dapps/src/tests/authorization.rs +++ b/dapps/src/tests/authorization.rs @@ -54,7 +54,7 @@ fn should_reject_on_invalid_auth() { // then assert_eq!(response.status, "HTTP/1.1 401 Unauthorized".to_owned()); - assert_eq!(response.body, "15\n

Unauthorized

\n0\n\n".to_owned()); + assert!(response.body.contains("Unauthorized")); assert_eq!(response.headers_raw.contains("WWW-Authenticate"), false); } diff --git a/dapps/src/tests/helpers.rs b/dapps/src/tests/helpers.rs index 84f638b34..1296162d2 100644 --- a/dapps/src/tests/helpers.rs +++ b/dapps/src/tests/helpers.rs @@ -15,16 +15,15 @@ // along with Parity. If not, see . use std::env; -use std::io::{Read, Write}; -use std::str::{self, Lines}; +use std::str; use std::sync::Arc; -use std::net::TcpStream; use rustc_serialize::hex::{ToHex, FromHex}; use ServerBuilder; use Server; use apps::urlhint::ContractClient; use util::{Bytes, Address, Mutex, ToPretty}; +use devtools::http_client; const REGISTRAR: &'static str = "8e4e9b13d4b45cb0befc93c3061b1408f67316b2"; const URLHINT: &'static str = "deadbeefcafe0000000000000000000000000000"; @@ -77,47 +76,6 @@ pub fn serve() -> Server { serve_hosts(None) } -pub struct Response { - pub status: String, - pub headers: Vec, - pub headers_raw: String, - pub body: String, +pub fn request(server: Server, request: &str) -> http_client::Response { + http_client::request(server.addr(), request) } - -pub fn read_block(lines: &mut Lines, all: bool) -> String { - let mut block = String::new(); - loop { - let line = lines.next(); - match line { - None => break, - Some("") if !all => break, - Some(v) => { - block.push_str(v); - block.push_str("\n"); - }, - } - } - block -} - -pub fn request(server: Server, request: &str) -> Response { - let mut req = TcpStream::connect(server.addr()).unwrap(); - req.write_all(request.as_bytes()).unwrap(); - - let mut response = String::new(); - req.read_to_string(&mut response).unwrap(); - - let mut lines = response.lines(); - let status = lines.next().unwrap().to_owned(); - let headers_raw = read_block(&mut lines, false); - let headers = headers_raw.split('\n').map(|v| v.to_owned()).collect(); - let body = read_block(&mut lines, true); - - Response { - status: status, - headers: headers, - headers_raw: headers_raw, - body: body, - } -} - diff --git a/dapps/src/tests/validation.rs b/dapps/src/tests/validation.rs index b233a07d8..b6a8c825c 100644 --- a/dapps/src/tests/validation.rs +++ b/dapps/src/tests/validation.rs @@ -34,7 +34,7 @@ fn should_reject_invalid_host() { // then assert_eq!(response.status, "HTTP/1.1 403 Forbidden".to_owned()); - assert_eq!(response.body, "85\n\n\t\t

Request with disallowed Host header has been blocked.

\n\t\t

Check the URL in your browser address bar.

\n\t\t\n0\n\n".to_owned()); + assert!(response.body.contains("Current host is disallowed")); } #[test] @@ -77,3 +77,24 @@ fn should_serve_dapps_domains() { assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned()); } +#[test] +// NOTE [todr] This is required for error pages to be styled properly. +fn should_allow_parity_utils_even_on_invalid_domain() { + // given + let server = serve_hosts(Some(vec!["localhost:8080".into()])); + + // when + let response = request(server, + "\ + GET /parity-utils/styles.css HTTP/1.1\r\n\ + Host: 127.0.0.1:8080\r\n\ + Connection: close\r\n\ + \r\n\ + {} + " + ); + + // then + assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned()); +} + diff --git a/devtools/src/http_client.rs b/devtools/src/http_client.rs new file mode 100644 index 000000000..27fa6ec50 --- /dev/null +++ b/devtools/src/http_client.rs @@ -0,0 +1,64 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::io::{Read, Write}; +use std::str::{self, Lines}; +use std::net::{TcpStream, SocketAddr}; + +pub struct Response { + pub status: String, + pub headers: Vec, + pub headers_raw: String, + pub body: String, +} + +pub fn read_block(lines: &mut Lines, all: bool) -> String { + let mut block = String::new(); + loop { + let line = lines.next(); + match line { + None => break, + Some("") if !all => break, + Some(v) => { + block.push_str(v); + block.push_str("\n"); + }, + } + } + block +} + +pub fn request(address: &SocketAddr, request: &str) -> Response { + let mut req = TcpStream::connect(address).unwrap(); + req.write_all(request.as_bytes()).unwrap(); + + let mut response = String::new(); + req.read_to_string(&mut response).unwrap(); + + let mut lines = response.lines(); + let status = lines.next().unwrap().to_owned(); + let headers_raw = read_block(&mut lines, false); + let headers = headers_raw.split('\n').map(|v| v.to_owned()).collect(); + let body = read_block(&mut lines, true); + + Response { + status: status, + headers: headers, + headers_raw: headers_raw, + body: body, + } +} + diff --git a/devtools/src/lib.rs b/devtools/src/lib.rs index 831e4315e..87457f7f3 100644 --- a/devtools/src/lib.rs +++ b/devtools/src/lib.rs @@ -22,6 +22,7 @@ extern crate rand; mod random_path; mod test_socket; mod stop_guard; +pub mod http_client; pub use random_path::*; pub use test_socket::*; diff --git a/signer/Cargo.toml b/signer/Cargo.toml index 5bb6325f1..cfcbedff8 100644 --- a/signer/Cargo.toml +++ b/signer/Cargo.toml @@ -19,6 +19,7 @@ ws = { git = "https://github.com/ethcore/ws-rs.git", branch = "mio-upstream-stab ethcore-util = { path = "../util" } ethcore-io = { path = "../util/io" } ethcore-rpc = { path = "../rpc" } +ethcore-devtools = { path = "../devtools" } parity-dapps-signer = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4", optional = true} clippy = { version = "0.0.85", optional = true} diff --git a/signer/src/lib.rs b/signer/src/lib.rs index 45383c7a1..abe84a03e 100644 --- a/signer/src/lib.rs +++ b/signer/src/lib.rs @@ -54,15 +54,13 @@ extern crate jsonrpc_core; extern crate ws; #[cfg(feature = "ui")] extern crate parity_dapps_signer as signer; +#[cfg(test)] +extern crate ethcore_devtools as devtools; mod authcode_store; mod ws_server; +#[cfg(test)] +mod tests; pub use authcode_store::*; pub use ws_server::*; - -#[cfg(test)] -mod tests { - #[test] - fn should_work() {} -} diff --git a/signer/src/tests/mod.rs b/signer/src/tests/mod.rs new file mode 100644 index 000000000..eaed49de8 --- /dev/null +++ b/signer/src/tests/mod.rs @@ -0,0 +1,81 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use std::env; +use std::thread; +use std::time::Duration; +use std::sync::Arc; +use devtools::http_client; +use rpc::ConfirmationsQueue; +use rand; + +use ServerBuilder; +use Server; + +pub fn serve() -> Server { + let queue = Arc::new(ConfirmationsQueue::default()); + let builder = ServerBuilder::new(queue, env::temp_dir()); + let port = 35000 + rand::random::() % 10000; + let res = builder.start(format!("127.0.0.1:{}", port).parse().unwrap()).unwrap(); + thread::sleep(Duration::from_millis(25)); + res +} + +pub fn request(server: Server, request: &str) -> http_client::Response { + http_client::request(server.addr(), request) +} + +#[test] +fn should_reject_invalid_host() { + // given + let server = serve(); + + // when + let response = request(server, + "\ + GET / HTTP/1.1\r\n\ + Host: test:8180\r\n\ + Connection: close\r\n\ + \r\n\ + {} + " + ); + + // then + assert_eq!(response.status, "HTTP/1.1 403 FORBIDDEN".to_owned()); + assert!(response.body.contains("URL Blocked")); +} + +#[test] +fn should_serve_styles_even_on_disallowed_domain() { + // given + let server = serve(); + + // when + let response = request(server, + "\ + GET /styles.css HTTP/1.1\r\n\ + Host: test:8180\r\n\ + Connection: close\r\n\ + \r\n\ + {} + " + ); + + // then + assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned()); +} + diff --git a/signer/src/ws_server/error_tpl.html b/signer/src/ws_server/error_tpl.html new file mode 100644 index 000000000..04a0f3c30 --- /dev/null +++ b/signer/src/ws_server/error_tpl.html @@ -0,0 +1,21 @@ + + + + + + {meta} + {title} + + + +
+
+

{title}

+

{message}

+

{details}

+
+
+ {version} +
+ + diff --git a/signer/src/ws_server/mod.rs b/signer/src/ws_server/mod.rs index 4ebe96bfa..6d332adbe 100644 --- a/signer/src/ws_server/mod.rs +++ b/signer/src/ws_server/mod.rs @@ -93,9 +93,15 @@ pub struct Server { broadcaster_handle: Option>, queue: Arc, panic_handler: Arc, + addr: SocketAddr, } impl Server { + /// Returns the address this server is listening on + pub fn addr(&self) -> &SocketAddr { + &self.addr + } + /// Starts a new `WebSocket` server in separate thread. /// Returns a `Server` handle which closes the server when droped. fn start(addr: SocketAddr, handler: Arc, queue: Arc, authcodes_path: PathBuf, skip_origin_validation: bool) -> Result { @@ -121,7 +127,7 @@ impl Server { // Spawn a thread with event loop let handle = thread::spawn(move || { ph.catch_panic(move || { - match ws.listen(addr).map_err(ServerError::from) { + match ws.listen(addr.clone()).map_err(ServerError::from) { Err(ServerError::IoError(io)) => die(format!( "Signer: Could not start listening on specified address. Make sure that no other instance is running on Signer's port. Details: {:?}", io @@ -158,6 +164,7 @@ impl Server { broadcaster_handle: Some(broadcaster_handle), queue: queue, panic_handler: panic_handler, + addr: addr, }) } } diff --git a/signer/src/ws_server/session.rs b/signer/src/ws_server/session.rs index 34097c24d..b53f47db8 100644 --- a/signer/src/ws_server/session.rs +++ b/signer/src/ws_server/session.rs @@ -107,10 +107,15 @@ impl ws::Handler for Session { fn on_request(&mut self, req: &ws::Request) -> ws::Result<(ws::Response)> { let origin = req.header("origin").or_else(|| req.header("Origin")).map(|x| &x[..]); let host = req.header("host").or_else(|| req.header("Host")).map(|x| &x[..]); + // Styles file is allowed for error pages to display nicely. + let is_styles_file = req.resource() == "/styles.css"; // Check request origin and host header. if !self.skip_origin_validation { - if !origin_is_allowed(&self.self_origin, origin) && !(origin.is_none() && origin_is_allowed(&self.self_origin, host)) { + let is_valid = origin_is_allowed(&self.self_origin, origin) || (origin.is_none() && origin_is_allowed(&self.self_origin, host)); + let is_valid = is_styles_file || is_valid; + + if !is_valid { warn!(target: "signer", "Blocked connection to Signer API from untrusted origin."); return Ok(error( ErrorType::Forbidden, @@ -121,8 +126,9 @@ impl ws::Handler for Session { } } - // Detect if it's a websocket request. - if req.header("sec-websocket-key").is_some() { + // Detect if it's a websocket request + // (styles file skips origin validation, so make sure to prevent WS connections on this resource) + if req.header("sec-websocket-key").is_some() && !is_styles_file { // Check authorization if !auth_is_valid(&self.authcodes_path, req.protocols()) { info!(target: "signer", "Unauthorized connection to Signer API blocked."); @@ -198,7 +204,7 @@ enum ErrorType { fn error(error: ErrorType, title: &str, message: &str, details: Option<&str>) -> ws::Response { let content = format!( - include_str!("../../../dapps/src/error_tpl.html"), + include_str!("./error_tpl.html"), title=title, meta="", message=message, From 89f1444c516aef931f90279d0df399a2c5fe8d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 1 Sep 2016 11:16:19 +0200 Subject: [PATCH 3/8] Displaying special page when syncing. 404 instead of redirection --- dapps/src/apps/fetcher.rs | 77 ++++++++++++++++++++++++---------- dapps/src/lib.rs | 26 ++++++++++-- dapps/src/router/mod.rs | 11 +++-- dapps/src/tests/fetch.rs | 38 +++++++++++++++++ dapps/src/tests/helpers.rs | 23 +++++++--- dapps/src/tests/mod.rs | 1 + dapps/src/tests/redirection.rs | 12 +++--- parity/dapps.rs | 11 +++-- parity/run.rs | 1 + 9 files changed, 157 insertions(+), 43 deletions(-) create mode 100644 dapps/src/tests/fetch.rs diff --git a/dapps/src/apps/fetcher.rs b/dapps/src/apps/fetcher.rs index 6223fb5e8..cb58af976 100644 --- a/dapps/src/apps/fetcher.rs +++ b/dapps/src/apps/fetcher.rs @@ -30,6 +30,7 @@ use hyper::Control; use hyper::status::StatusCode; use random_filename; +use SyncStatus; use util::{Mutex, H256}; use util::sha3::sha3; use page::LocalPageEndpoint; @@ -44,6 +45,7 @@ const MAX_CACHED_DAPPS: usize = 10; pub struct AppFetcher { dapps_path: PathBuf, resolver: R, + sync: Arc, dapps: Arc>, } @@ -56,13 +58,14 @@ impl Drop for AppFetcher { impl AppFetcher { - pub fn new(resolver: R) -> Self { + pub fn new(resolver: R, sync_status: Arc) -> Self { let mut dapps_path = env::temp_dir(); dapps_path.push(random_filename()); AppFetcher { dapps_path: dapps_path, resolver: resolver, + sync: sync_status, dapps: Arc::new(Mutex::new(ContentCache::default())), } } @@ -74,14 +77,20 @@ impl AppFetcher { pub fn contains(&self, app_id: &str) -> bool { let mut dapps = self.dapps.lock(); - match dapps.get(app_id) { - // Check if we already have the app - Some(_) => true, - // fallback to resolver - None => match app_id.from_hex() { - Ok(app_id) => self.resolver.resolve(app_id).is_some(), - _ => false, - }, + // Check if we already have the app + if dapps.get(app_id).is_some() { + return true; + } + // fallback to resolver + if let Ok(app_id) = app_id.from_hex() { + // if app_id is valid, but we are syncing always return true. + if self.sync.is_major_syncing() { + return true; + } + // else try to resolve the app_id + self.resolver.resolve(app_id).is_some() + } else { + false } } @@ -89,6 +98,15 @@ impl AppFetcher { let mut dapps = self.dapps.lock(); let app_id = path.app_id.clone(); + if self.sync.is_major_syncing() { + return Box::new(ContentHandler::error( + StatusCode::ServiceUnavailable, + "Sync in progress", + "Your node is still syncing. We cannot resolve any content before it's fully synced.", + None + )); + } + let (new_status, handler) = { let status = dapps.get(&app_id); match status { @@ -108,20 +126,32 @@ impl AppFetcher { // We need to start fetching app None => { let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true."); - let app = self.resolver.resolve(app_hex).expect("to_handler is called only when `contains` returns true."); - let abort = Arc::new(AtomicBool::new(false)); + let app = self.resolver.resolve(app_hex); - (Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new( - app, - abort, - control, - path.using_dapps_domains, - DappInstaller { - dapp_id: app_id.clone(), - dapps_path: self.dapps_path.clone(), - dapps: self.dapps.clone(), - } - )) as Box) + if let Some(app) = app { + let abort = Arc::new(AtomicBool::new(false)); + + (Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new( + app, + abort, + control, + path.using_dapps_domains, + DappInstaller { + dapp_id: app_id.clone(), + dapps_path: self.dapps_path.clone(), + dapps: self.dapps.clone(), + } + )) as Box) + } else { + // This may happen when sync status changes in between + // `contains` and `to_handler` + (None, Box::new(ContentHandler::error( + StatusCode::NotFound, + "Resource Not Found", + "Requested resource was not found.", + None + )) as Box) + } }, } }; @@ -274,6 +304,7 @@ impl ContentValidator for DappInstaller { #[cfg(test)] mod tests { use std::env; + use std::sync::Arc; use util::Bytes; use endpoint::EndpointInfo; use page::LocalPageEndpoint; @@ -292,7 +323,7 @@ mod tests { fn should_true_if_contains_the_app() { // given let path = env::temp_dir(); - let fetcher = AppFetcher::new(FakeResolver); + let fetcher = AppFetcher::new(FakeResolver, Arc::new(|| false)); let handler = LocalPageEndpoint::new(path, EndpointInfo { name: "fake".into(), description: "".into(), diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index e24def0a1..0a2297189 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -88,11 +88,22 @@ use ethcore_rpc::Extendable; static DAPPS_DOMAIN : &'static str = ".parity"; +/// Indicates sync status +pub trait SyncStatus: Send + Sync { + /// Returns true if there is a major sync happening. + fn is_major_syncing(&self) -> bool; +} + +impl SyncStatus for F where F: Fn() -> bool + Send + Sync { + fn is_major_syncing(&self) -> bool { self() } +} + /// Webapps HTTP+RPC server build. pub struct ServerBuilder { dapps_path: String, handler: Arc, registrar: Arc, + sync_status: Arc, } impl Extendable for ServerBuilder { @@ -108,9 +119,15 @@ impl ServerBuilder { dapps_path: dapps_path, handler: Arc::new(IoHandler::new()), registrar: registrar, + sync_status: Arc::new(|| false), } } + /// Change default sync status. + pub fn with_sync_status(&mut self, status: Arc) { + self.sync_status = status; + } + /// Asynchronously start server with no authentication, /// returns result with `Server` handle on success or an error. pub fn start_unsecured_http(&self, addr: &SocketAddr, hosts: Option>) -> Result { @@ -120,7 +137,8 @@ impl ServerBuilder { NoAuth, self.handler.clone(), self.dapps_path.clone(), - self.registrar.clone() + self.registrar.clone(), + self.sync_status.clone(), ) } @@ -133,7 +151,8 @@ impl ServerBuilder { HttpBasicAuth::single_user(username, password), self.handler.clone(), self.dapps_path.clone(), - self.registrar.clone() + self.registrar.clone(), + self.sync_status.clone(), ) } } @@ -167,10 +186,11 @@ impl Server { handler: Arc, dapps_path: String, registrar: Arc, + sync_status: Arc, ) -> Result { let panic_handler = Arc::new(Mutex::new(None)); let authorization = Arc::new(authorization); - let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::new(apps::urlhint::URLHintContract::new(registrar))); + let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status)); let endpoints = Arc::new(apps::all_endpoints(dapps_path)); let special = Arc::new({ let mut special = HashMap::new(); diff --git a/dapps/src/router/mod.rs b/dapps/src/router/mod.rs index 9326917bf..5dc1018b0 100644 --- a/dapps/src/router/mod.rs +++ b/dapps/src/router/mod.rs @@ -24,12 +24,12 @@ use DAPPS_DOMAIN; use std::sync::Arc; use std::collections::HashMap; use url::{Url, Host}; -use hyper::{self, server, Next, Encoder, Decoder, Control}; +use hyper::{self, server, Next, Encoder, Decoder, Control, StatusCode}; use hyper::net::HttpStream; use apps; use apps::fetcher::AppFetcher; use endpoint::{Endpoint, Endpoints, EndpointPath}; -use handlers::{Redirection, extract_url}; +use handlers::{Redirection, extract_url, ContentHandler}; use self::auth::{Authorization, Authorized}; /// Special endpoints are accessible on every domain (every dapp) @@ -94,7 +94,12 @@ impl server::Handler for Router { // Redirection to main page (maybe 404 instead?) (Some(ref path), _) if *req.method() == hyper::method::Method::Get => { let address = apps::redirection_address(path.using_dapps_domains, self.main_page); - Redirection::new(address.as_str()) + Box::new(ContentHandler::error( + StatusCode::NotFound, + "404 Not Found", + "Requested content was not found on a server.", + Some(&format!("Go back to the Home Page.", address)) + )) }, // Redirect any GET request to home. _ if *req.method() == hyper::method::Method::Get => { diff --git a/dapps/src/tests/fetch.rs b/dapps/src/tests/fetch.rs new file mode 100644 index 000000000..6fd65c00f --- /dev/null +++ b/dapps/src/tests/fetch.rs @@ -0,0 +1,38 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use tests::helpers::{serve_with_registrar, request}; + +#[test] +fn should_resolve_dapp() { + // given + let (server, registrar) = serve_with_registrar(); + + // when + let response = request(server, + "\ + GET / HTTP/1.1\r\n\ + Host: 1472a9e190620cdf6b31f383373e45efcfe869a820c91f9ccd7eb9fb45e4985d.parity\r\n\ + Connection: close\r\n\ + \r\n\ + " + ); + + // then + assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned()); + assert_eq!(registrar.calls.lock().len(), 2); +} + diff --git a/dapps/src/tests/helpers.rs b/dapps/src/tests/helpers.rs index 1296162d2..4cd21520c 100644 --- a/dapps/src/tests/helpers.rs +++ b/dapps/src/tests/helpers.rs @@ -58,22 +58,35 @@ impl ContractClient for FakeRegistrar { } } -pub fn serve_hosts(hosts: Option>) -> Server { +pub fn init_server(hosts: Option>) -> (Server, Arc) { let registrar = Arc::new(FakeRegistrar::new()); let mut dapps_path = env::temp_dir(); dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading"); - let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar); - builder.start_unsecured_http(&"127.0.0.1:0".parse().unwrap(), hosts).unwrap() + let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar.clone()); + ( + builder.start_unsecured_http(&"127.0.0.1:0".parse().unwrap(), hosts).unwrap(), + registrar, + ) } pub fn serve_with_auth(user: &str, pass: &str) -> Server { let registrar = Arc::new(FakeRegistrar::new()); - let builder = ServerBuilder::new(env::temp_dir().to_str().unwrap().into(), registrar); + let mut dapps_path = env::temp_dir(); + dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading"); + let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar); builder.start_basic_auth_http(&"127.0.0.1:0".parse().unwrap(), None, user, pass).unwrap() } +pub fn serve_hosts(hosts: Option>) -> Server { + init_server(hosts).0 +} + +pub fn serve_with_registrar() -> (Server, Arc) { + init_server(None) +} + pub fn serve() -> Server { - serve_hosts(None) + init_server(None).0 } pub fn request(server: Server, request: &str) -> http_client::Response { diff --git a/dapps/src/tests/mod.rs b/dapps/src/tests/mod.rs index 8c5bf2283..cc0e693c9 100644 --- a/dapps/src/tests/mod.rs +++ b/dapps/src/tests/mod.rs @@ -20,6 +20,7 @@ mod helpers; mod api; mod authorization; +mod fetch; mod redirection; mod validation; diff --git a/dapps/src/tests/redirection.rs b/dapps/src/tests/redirection.rs index 53aa393e2..2fdecbd4d 100644 --- a/dapps/src/tests/redirection.rs +++ b/dapps/src/tests/redirection.rs @@ -57,7 +57,7 @@ fn should_redirect_to_home_when_trailing_slash_is_missing() { } #[test] -fn should_redirect_to_home_on_invalid_dapp() { +fn should_display_404_on_invalid_dapp() { // given let server = serve(); @@ -72,12 +72,12 @@ fn should_redirect_to_home_on_invalid_dapp() { ); // then - assert_eq!(response.status, "HTTP/1.1 302 Found".to_owned()); - assert_eq!(response.headers.get(0).unwrap(), "Location: /home/"); + assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned()); + assert!(response.body.contains("href=\"/home/")); } #[test] -fn should_redirect_to_home_on_invalid_dapp_with_domain() { +fn should_display_404_on_invalid_dapp_with_domain() { // given let server = serve(); @@ -92,8 +92,8 @@ fn should_redirect_to_home_on_invalid_dapp_with_domain() { ); // then - assert_eq!(response.status, "HTTP/1.1 302 Found".to_owned()); - assert_eq!(response.headers.get(0).unwrap(), "Location: http://home.parity/"); + assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned()); + assert!(response.body.contains("href=\"http://home.parity/")); } #[test] diff --git a/parity/dapps.rs b/parity/dapps.rs index 7f759ed3c..fae395b5a 100644 --- a/parity/dapps.rs +++ b/parity/dapps.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use io::PanicHandler; use rpc_apis; use ethcore::client::Client; +use ethsync::SyncProvider; use helpers::replace_home; #[derive(Debug, PartialEq, Clone)] @@ -49,6 +50,7 @@ pub struct Dependencies { pub panic_handler: Arc, pub apis: Arc, pub client: Arc, + pub sync: Arc, } pub fn new(configuration: Configuration, deps: Dependencies) -> Result, String> { @@ -117,9 +119,12 @@ mod server { ) -> Result { use ethcore_dapps as dapps; - let server = dapps::ServerBuilder::new(dapps_path, Arc::new(Registrar { - client: deps.client.clone(), - })); + let mut server = dapps::ServerBuilder::new( + dapps_path, + Arc::new(Registrar { client: deps.client.clone() }) + ); + let sync = deps.sync.clone(); + server.with_sync_status(Arc::new(move || sync.status().is_major_syncing())); let server = rpc_apis::setup_rpc(server, deps.apis.clone(), rpc_apis::ApiSet::UnsafeContext); let start_result = match auth { None => { diff --git a/parity/run.rs b/parity/run.rs index 71995cd5f..8a68fe1af 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -224,6 +224,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> { panic_handler: panic_handler.clone(), apis: deps_for_rpc_apis.clone(), client: client.clone(), + sync: sync_provider.clone(), }; // start dapps server From 055ff91464f746845f1092e542c1912a6afbd68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 1 Sep 2016 11:29:40 +0200 Subject: [PATCH 4/8] Bumping ui --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9ad5d6bd..c5df53681 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,7 +1070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "parity-dapps" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" +source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc" dependencies = [ "aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1084,7 +1084,7 @@ dependencies = [ [[package]] name = "parity-dapps-home" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" +source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1092,7 +1092,7 @@ dependencies = [ [[package]] name = "parity-dapps-signer" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" +source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1100,7 +1100,7 @@ dependencies = [ [[package]] name = "parity-dapps-status" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" +source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] @@ -1108,7 +1108,7 @@ dependencies = [ [[package]] name = "parity-dapps-wallet" version = "1.4.0" -source = "git+https://github.com/ethcore/parity-ui.git#be8754c3ccd142c43a20b82e9b1fba416b5b332a" +source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc" dependencies = [ "parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", ] From a9bc021022414e64e13bde9b20a0a76ecf77ba21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 1 Sep 2016 11:54:09 +0200 Subject: [PATCH 5/8] 404 pages for dapps resources --- dapps/src/page/builtin.rs | 2 +- dapps/src/page/handler.rs | 52 ++++++++++++++++++++++++++++----------- dapps/src/page/local.rs | 2 +- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/dapps/src/page/builtin.rs b/dapps/src/page/builtin.rs index 1c7ca32d4..3c8d66686 100644 --- a/dapps/src/page/builtin.rs +++ b/dapps/src/page/builtin.rs @@ -79,7 +79,7 @@ impl Endpoint for PageEndpoint { app: BuiltinDapp::new(self.app.clone()), prefix: self.prefix.clone(), path: path, - file: None, + file: Default::default(), safe_to_embed: self.safe_to_embed, }) } diff --git a/dapps/src/page/handler.rs b/dapps/src/page/handler.rs index eca242e7b..70487443e 100644 --- a/dapps/src/page/handler.rs +++ b/dapps/src/page/handler.rs @@ -22,6 +22,7 @@ use hyper::net::HttpStream; use hyper::status::StatusCode; use hyper::{Decoder, Encoder, Next}; use endpoint::EndpointPath; +use handlers::ContentHandler; /// Represents a file that can be sent to client. /// Implementation should keep track of bytes already sent internally. @@ -48,6 +49,25 @@ pub trait Dapp: Send + 'static { fn file(&self, path: &str) -> Option; } +/// Currently served by `PageHandler` file +pub enum ServedFile { + /// File from dapp + File(T::DappFile), + /// Error (404) + Error(ContentHandler), +} + +impl Default for ServedFile { + fn default() -> Self { + ServedFile::Error(ContentHandler::error( + StatusCode::NotFound, + "404 Not Found", + "Requested dapp resource was not found.", + None + )) + } +} + /// A handler for a single webapp. /// Resolves correct paths and serves as a plumbing code between /// hyper server and dapp. @@ -55,7 +75,7 @@ pub struct PageHandler { /// A Dapp. pub app: T, /// File currently being served (or `None` if file does not exist). - pub file: Option, + pub file: ServedFile, /// Optional prefix to strip from path. pub prefix: Option, /// Requested path. @@ -95,7 +115,7 @@ impl server::Handler for PageHandler { self.app.file(&self.extract_path(url.path())) }, _ => None, - }; + }.map_or_else(|| ServedFile::default(), |f| ServedFile::File(f)); Next::write() } @@ -104,24 +124,26 @@ impl server::Handler for PageHandler { } fn on_response(&mut self, res: &mut server::Response) -> Next { - if let Some(ref f) = self.file { - res.set_status(StatusCode::Ok); - res.headers_mut().set(header::ContentType(f.content_type().parse().unwrap())); - if !self.safe_to_embed { - res.headers_mut().set_raw("X-Frame-Options", vec![b"SAMEORIGIN".to_vec()]); + match self.file { + ServedFile::File(ref f) => { + res.set_status(StatusCode::Ok); + res.headers_mut().set(header::ContentType(f.content_type().parse().unwrap())); + if !self.safe_to_embed { + res.headers_mut().set_raw("X-Frame-Options", vec![b"SAMEORIGIN".to_vec()]); + } + Next::write() + }, + ServedFile::Error(ref mut handler) => { + handler.on_response(res) } - Next::write() - } else { - res.set_status(StatusCode::NotFound); - Next::write() } } fn on_response_writable(&mut self, encoder: &mut Encoder) -> Next { match self.file { - None => Next::end(), - Some(ref f) if f.is_drained() => Next::end(), - Some(ref mut f) => match encoder.write(f.next_chunk()) { + ServedFile::Error(ref mut handler) => handler.on_response_writable(encoder), + ServedFile::File(ref f) if f.is_drained() => Next::end(), + ServedFile::File(ref mut f) => match encoder.write(f.next_chunk()) { Ok(bytes) => { f.bytes_written(bytes); Next::write() @@ -190,7 +212,7 @@ fn should_extract_path_with_appid() { port: 8080, using_dapps_domains: true, }, - file: None, + file: Default::default(), safe_to_embed: true, }; diff --git a/dapps/src/page/local.rs b/dapps/src/page/local.rs index dcfd9bed2..86d4273d5 100644 --- a/dapps/src/page/local.rs +++ b/dapps/src/page/local.rs @@ -49,7 +49,7 @@ impl Endpoint for LocalPageEndpoint { app: LocalDapp::new(self.path.clone()), prefix: None, path: path, - file: None, + file: Default::default(), safe_to_embed: false, }) } From 9f8482e96843f077a8f80b63e3a9cf6276800040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 1 Sep 2016 15:33:26 +0200 Subject: [PATCH 6/8] Consistent capitalization of titles --- dapps/src/apps/fetcher.rs | 4 ++-- dapps/src/handlers/fetch.rs | 2 +- dapps/src/router/auth.rs | 4 +++- dapps/src/router/host_validation.rs | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dapps/src/apps/fetcher.rs b/dapps/src/apps/fetcher.rs index 869a34df2..7d7db7b8b 100644 --- a/dapps/src/apps/fetcher.rs +++ b/dapps/src/apps/fetcher.rs @@ -101,9 +101,9 @@ impl AppFetcher { if self.sync.is_major_syncing() { return Box::new(ContentHandler::error( StatusCode::ServiceUnavailable, - "Sync in progress", + "Sync In Progress", "Your node is still syncing. We cannot resolve any content before it's fully synced.", - None + Some("Refresh") )); } diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index 7cbb2537b..ef37cfc61 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -133,7 +133,7 @@ impl server::Handler for ContentFetcherHandler< Err(e) => FetchState::Error(ContentHandler::error( StatusCode::BadGateway, "Unable To Start Dapp Download", - "Could not initialize download of the dapp. It might be a problem with remote server.", + "Could not initialize download of the dapp. It might be a problem with the remote server.", Some(&format!("{}", e)), )), } diff --git a/dapps/src/router/auth.rs b/dapps/src/router/auth.rs index 596796eed..ff05420bc 100644 --- a/dapps/src/router/auth.rs +++ b/dapps/src/router/auth.rs @@ -57,7 +57,9 @@ impl Authorization for HttpBasicAuth { Access::Denied => { Authorized::No(Box::new(ContentHandler::error( status::StatusCode::Unauthorized, - "Unauthorized", "You need to provide valid credentials to access this page.", None + "Unauthorized", + "You need to provide valid credentials to access this page.", + None ))) }, Access::AuthRequired => { diff --git a/dapps/src/router/host_validation.rs b/dapps/src/router/host_validation.rs index d1d651c5d..5be30ef8b 100644 --- a/dapps/src/router/host_validation.rs +++ b/dapps/src/router/host_validation.rs @@ -39,7 +39,7 @@ pub fn is_valid(request: &server::Request, allowed_hosts: &[String], pub fn host_invalid_response() -> Box + Send> { Box::new(ContentHandler::error(StatusCode::Forbidden, - "Current host is disallowed", + "Current Host Is Disallowed", "You are trying to access your node using incorrect address.", Some("Use allowed URL or specify different hosts CLI options.") )) From d0bc80e58a7fe34ca03021ab8f31f8a333493cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 2 Sep 2016 10:10:51 +0200 Subject: [PATCH 7/8] Fixign tests --- dapps/src/tests/api.rs | 2 +- dapps/src/tests/authorization.rs | 2 +- dapps/src/tests/validation.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dapps/src/tests/api.rs b/dapps/src/tests/api.rs index a9d3eba3b..ab0d33726 100644 --- a/dapps/src/tests/api.rs +++ b/dapps/src/tests/api.rs @@ -57,7 +57,7 @@ fn should_serve_apps() { // then assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned()); assert_eq!(response.headers.get(0).unwrap(), "Content-Type: application/json"); - assert!(response.body.contains("Parity Home Screen")); + assert!(response.body.contains("Parity Home Screen"), response.body); } #[test] diff --git a/dapps/src/tests/authorization.rs b/dapps/src/tests/authorization.rs index e7a70c9cd..9d5a46af4 100644 --- a/dapps/src/tests/authorization.rs +++ b/dapps/src/tests/authorization.rs @@ -54,7 +54,7 @@ fn should_reject_on_invalid_auth() { // then assert_eq!(response.status, "HTTP/1.1 401 Unauthorized".to_owned()); - assert!(response.body.contains("Unauthorized")); + assert!(response.body.contains("Unauthorized"), response.body); assert_eq!(response.headers_raw.contains("WWW-Authenticate"), false); } diff --git a/dapps/src/tests/validation.rs b/dapps/src/tests/validation.rs index b6a8c825c..c39350cce 100644 --- a/dapps/src/tests/validation.rs +++ b/dapps/src/tests/validation.rs @@ -34,7 +34,7 @@ fn should_reject_invalid_host() { // then assert_eq!(response.status, "HTTP/1.1 403 Forbidden".to_owned()); - assert!(response.body.contains("Current host is disallowed")); + assert!(response.body.contains("Current Host Is Disallowed"), response.body); } #[test] From 5c5d9c8ccdd4ba84cee53aadf6512a1cb1ea5d94 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Tue, 6 Sep 2016 15:31:13 +0200 Subject: [PATCH 8/8] Snapshot sync (#2047) * PV64 sync * Tests * Client DB restore * Snapshot restoration over IPC * Upating test * Minor tweaks * Upating test --- ethcore/build.rs | 1 + ethcore/src/block_queue.rs | 2 +- ethcore/src/blockchain/config.rs | 2 +- ethcore/src/client/client.rs | 199 +++++---- ethcore/src/client/config.rs | 2 +- ethcore/src/miner/miner.rs | 5 + ethcore/src/service.rs | 21 +- ethcore/src/snapshot/mod.rs | 64 +-- ethcore/src/snapshot/service.rs | 123 ++---- .../src/snapshot/snapshot_service_trait.rs | 54 +++ ethcore/src/types/mod.rs.in | 2 + ethcore/src/types/restoration_status.rs | 34 ++ ethcore/src/types/snapshot_manifest.rs | 70 ++++ ethcore/src/verification/mod.rs | 2 +- parity/main.rs | 22 +- parity/modules.rs | 7 +- parity/run.rs | 3 +- parity/snapshot.rs | 7 +- parity/sync.rs | 4 +- rpc/src/v1/impls/eth.rs | 3 +- rpc/src/v1/tests/helpers/sync_provider.rs | 2 + sync/src/api.rs | 27 +- sync/src/chain.rs | 387 +++++++++++++++--- sync/src/lib.rs | 36 +- sync/src/snapshot.rs | 200 +++++++++ sync/src/sync_io.rs | 20 +- sync/src/tests/helpers.rs | 29 +- sync/src/tests/mod.rs | 1 + sync/src/tests/snapshot.rs | 123 ++++++ util/network/src/host.rs | 6 + util/network/src/session.rs | 5 + util/src/kvdb.rs | 215 +++++++--- 32 files changed, 1258 insertions(+), 420 deletions(-) create mode 100644 ethcore/src/snapshot/snapshot_service_trait.rs create mode 100644 ethcore/src/types/restoration_status.rs create mode 100644 ethcore/src/types/snapshot_manifest.rs create mode 100644 sync/src/snapshot.rs create mode 100644 sync/src/tests/snapshot.rs diff --git a/ethcore/build.rs b/ethcore/build.rs index 2e07cbc2f..b83955708 100644 --- a/ethcore/build.rs +++ b/ethcore/build.rs @@ -19,5 +19,6 @@ extern crate ethcore_ipc_codegen; fn main() { ethcore_ipc_codegen::derive_binary("src/types/mod.rs.in").unwrap(); ethcore_ipc_codegen::derive_ipc("src/client/traits.rs").unwrap(); + ethcore_ipc_codegen::derive_ipc("src/snapshot/snapshot_service_trait.rs").unwrap(); ethcore_ipc_codegen::derive_ipc("src/client/chain_notify.rs").unwrap(); } diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 7d686cec0..c441136fd 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -37,7 +37,7 @@ const MIN_MEM_LIMIT: usize = 16384; const MIN_QUEUE_LIMIT: usize = 512; /// Block queue configuration -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct BlockQueueConfig { /// Maximum number of blocks to keep in unverified queue. /// When the limit is reached, is_full returns true. diff --git a/ethcore/src/blockchain/config.rs b/ethcore/src/blockchain/config.rs index 1a0ab9d42..324474958 100644 --- a/ethcore/src/blockchain/config.rs +++ b/ethcore/src/blockchain/config.rs @@ -17,7 +17,7 @@ //! Blockchain configuration. /// Blockchain configuration. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct Config { /// Preferred cache size in bytes. pub pref_cache_size: usize, diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 554afab38..dfb4f3ad3 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -32,7 +32,7 @@ use util::kvdb::*; // other use io::*; use views::{BlockView, HeaderView, BodyView}; -use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult}; +use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError}; use header::BlockNumber; use state::State; use spec::Spec; @@ -122,11 +122,13 @@ impl SleepState { /// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue. pub struct Client { mode: Mode, - chain: Arc, - tracedb: Arc>, + chain: RwLock>, + tracedb: RwLock>, engine: Arc, - db: Arc, - state_db: Mutex>, + config: ClientConfig, + db: RwLock>, + pruning: journaldb::Algorithm, + state_db: RwLock>, block_queue: BlockQueue, report: RwLock, import_lock: Mutex<()>, @@ -168,8 +170,8 @@ impl Client { db_config.wal = config.db_wal; let db = Arc::new(try!(Database::open(&db_config, &path.to_str().unwrap()).map_err(ClientError::Database))); - let chain = Arc::new(BlockChain::new(config.blockchain, &gb, db.clone())); - let tracedb = Arc::new(try!(TraceDB::new(config.tracing, db.clone(), chain.clone()))); + let chain = Arc::new(BlockChain::new(config.blockchain.clone(), &gb, db.clone())); + let tracedb = RwLock::new(try!(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone()))); let mut state_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE); if state_db.is_empty() && try!(spec.ensure_db_good(state_db.as_hashdb_mut())) { @@ -184,32 +186,34 @@ impl Client { let engine = spec.engine.clone(); - let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel.clone()); + let block_queue = BlockQueue::new(config.queue.clone(), engine.clone(), message_channel.clone()); let panic_handler = PanicHandler::new_in_arc(); panic_handler.forward_from(&block_queue); let awake = match config.mode { Mode::Dark(..) => false, _ => true }; let factories = Factories { - vm: EvmFactory::new(config.vm_type), - trie: TrieFactory::new(config.trie_spec), + vm: EvmFactory::new(config.vm_type.clone()), + trie: TrieFactory::new(config.trie_spec.clone()), accountdb: Default::default(), }; let client = Client { sleep_state: Mutex::new(SleepState::new(awake)), liveness: AtomicBool::new(awake), - mode: config.mode, - chain: chain, + mode: config.mode.clone(), + chain: RwLock::new(chain), tracedb: tracedb, engine: engine, - db: db, - state_db: Mutex::new(state_db), + pruning: config.pruning.clone(), + verifier: verification::new(config.verifier_type.clone()), + config: config, + db: RwLock::new(db), + state_db: RwLock::new(state_db), block_queue: block_queue, report: RwLock::new(Default::default()), import_lock: Mutex::new(()), panic_handler: panic_handler, - verifier: verification::new(config.verifier_type), miner: miner, io_channel: message_channel, notify: RwLock::new(Vec::new()), @@ -253,8 +257,9 @@ impl Client { let mut last_hashes = LastHashes::new(); last_hashes.resize(256, H256::default()); last_hashes[0] = parent_hash; + let chain = self.chain.read(); for i in 0..255 { - match self.chain.block_details(&last_hashes[i]) { + match chain.block_details(&last_hashes[i]) { Some(details) => { last_hashes[i + 1] = details.parent.clone(); }, @@ -270,22 +275,23 @@ impl Client { let engine = &*self.engine; let header = &block.header; + let chain = self.chain.read(); // Check the block isn't so old we won't be able to enact it. - let best_block_number = self.chain.best_block_number(); + let best_block_number = chain.best_block_number(); if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY { warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number); return Err(()); } // Verify Block Family - let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &*self.chain); + let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &**chain); if let Err(e) = verify_family_result { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); }; // Check if Parent is in chain - let chain_has_parent = self.chain.block_header(header.parent_hash()); + let chain_has_parent = chain.block_header(header.parent_hash()); if let None = chain_has_parent { warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash()); return Err(()); @@ -294,9 +300,9 @@ impl Client { // Enact Verified Block let parent = chain_has_parent.unwrap(); let last_hashes = self.build_last_hashes(header.parent_hash().clone()); - let db = self.state_db.lock().boxed_clone(); + let db = self.state_db.read().boxed_clone(); - let enact_result = enact_verified(block, engine, self.tracedb.tracing_enabled(), db, &parent, last_hashes, self.factories.clone()); + let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone()); if let Err(e) = enact_result { warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); return Err(()); @@ -408,17 +414,18 @@ impl Client { } } - self.db.flush().expect("DB flush failed."); + self.db.read().flush().expect("DB flush failed."); imported } fn commit_block(&self, block: B, hash: &H256, block_data: &[u8]) -> ImportRoute where B: IsBlock + Drain { let number = block.header().number(); let parent = block.header().parent_hash().clone(); + let chain = self.chain.read(); // Are we committing an era? let ancient = if number >= HISTORY { let n = number - HISTORY; - Some((n, self.chain.block_hash(n).unwrap())) + Some((n, chain.block_hash(n).unwrap())) } else { None }; @@ -432,14 +439,14 @@ impl Client { //let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new)); - let mut batch = DBTransaction::new(&self.db); + let mut batch = DBTransaction::new(&self.db.read()); // CHECK! I *think* this is fine, even if the state_root is equal to another // already-imported block of the same number. // TODO: Prove it with a test. block.drain().commit(&mut batch, number, hash, ancient).expect("DB commit failed."); - let route = self.chain.insert_block(&mut batch, block_data, receipts); - self.tracedb.import(&mut batch, TraceImportRequest { + let route = chain.insert_block(&mut batch, block_data, receipts); + self.tracedb.read().import(&mut batch, TraceImportRequest { traces: traces.into(), block_hash: hash.clone(), block_number: number, @@ -447,8 +454,8 @@ impl Client { retracted: route.retracted.len() }); // Final commit to the DB - self.db.write_buffered(batch); - self.chain.commit(); + self.db.read().write_buffered(batch); + chain.commit(); self.update_last_hashes(&parent, hash); route @@ -491,10 +498,10 @@ impl Client { }; self.block_header(id).and_then(|header| { - let db = self.state_db.lock().boxed_clone(); + let db = self.state_db.read().boxed_clone(); // early exit for pruned blocks - if db.is_pruned() && self.chain.best_block_number() >= block_number + HISTORY { + if db.is_pruned() && self.chain.read().best_block_number() >= block_number + HISTORY { return None; } @@ -522,7 +529,7 @@ impl Client { /// Get a copy of the best block's state. pub fn state(&self) -> State { State::from_existing( - self.state_db.lock().boxed_clone(), + self.state_db.read().boxed_clone(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce(), self.factories.clone()) @@ -531,22 +538,22 @@ impl Client { /// Get info on the cache. pub fn blockchain_cache_info(&self) -> BlockChainCacheSize { - self.chain.cache_size() + self.chain.read().cache_size() } /// Get the report. pub fn report(&self) -> ClientReport { let mut report = self.report.read().clone(); - report.state_db_mem = self.state_db.lock().mem_used(); + report.state_db_mem = self.state_db.read().mem_used(); report } /// Tick the client. // TODO: manage by real events. pub fn tick(&self) { - self.chain.collect_garbage(); + self.chain.read().collect_garbage(); self.block_queue.collect_garbage(); - self.tracedb.collect_garbage(); + self.tracedb.read().collect_garbage(); match self.mode { Mode::Dark(timeout) => { @@ -584,16 +591,16 @@ impl Client { pub fn block_number(&self, id: BlockID) -> Option { match id { BlockID::Number(number) => Some(number), - BlockID::Hash(ref hash) => self.chain.block_number(hash), + BlockID::Hash(ref hash) => self.chain.read().block_number(hash), BlockID::Earliest => Some(0), - BlockID::Latest | BlockID::Pending => Some(self.chain.best_block_number()), + BlockID::Latest | BlockID::Pending => Some(self.chain.read().best_block_number()), } } /// Take a snapshot at the given block. /// If the ID given is "latest", this will default to 1000 blocks behind. - pub fn take_snapshot(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), ::error::Error> { - let db = self.state_db.lock().boxed_clone(); + pub fn take_snapshot(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), EthcoreError> { + let db = self.state_db.read().boxed_clone(); let best_block_number = self.chain_info().best_block_number; let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at))); @@ -618,7 +625,7 @@ impl Client { }, }; - try!(snapshot::take_snapshot(&self.chain, start_hash, db.as_hashdb(), writer, p)); + try!(snapshot::take_snapshot(&self.chain.read(), start_hash, db.as_hashdb(), writer, p)); Ok(()) } @@ -634,8 +641,8 @@ impl Client { fn transaction_address(&self, id: TransactionID) -> Option { match id { - TransactionID::Hash(ref hash) => self.chain.transaction_address(hash), - TransactionID::Location(id, index) => Self::block_hash(&self.chain, id).map(|hash| TransactionAddress { + TransactionID::Hash(ref hash) => self.chain.read().transaction_address(hash), + TransactionID::Location(id, index) => Self::block_hash(&self.chain.read(), id).map(|hash| TransactionAddress { block_hash: hash, index: index, }) @@ -666,6 +673,25 @@ impl Client { } } +impl snapshot::DatabaseRestore for Client { + /// Restart the client with a new backend + fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError> { + let _import_lock = self.import_lock.lock(); + let mut state_db = self.state_db.write(); + let mut chain = self.chain.write(); + let mut tracedb = self.tracedb.write(); + self.miner.clear(); + let db = self.db.write(); + try!(db.restore(new_db)); + + *state_db = journaldb::new(db.clone(), self.pruning, ::db::COL_STATE); + *chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone())); + *tracedb = try!(TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone()).map_err(ClientError::from)); + Ok(()) + } +} + + impl BlockChainClient for Client { fn call(&self, t: &SignedTransaction, block: BlockID, analytics: CallAnalytics) -> Result { let header = try!(self.block_header(block).ok_or(CallError::StatePruned)); @@ -749,15 +775,17 @@ impl BlockChainClient for Client { } fn best_block_header(&self) -> Bytes { - self.chain.best_block_header() + self.chain.read().best_block_header() } fn block_header(&self, id: BlockID) -> Option { - Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_header_data(&hash)) + let chain = self.chain.read(); + Self::block_hash(&chain, id).and_then(|hash| chain.block_header_data(&hash)) } fn block_body(&self, id: BlockID) -> Option { - Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_body(&hash)) + let chain = self.chain.read(); + Self::block_hash(&chain, id).and_then(|hash| chain.block_body(&hash)) } fn block(&self, id: BlockID) -> Option { @@ -766,14 +794,16 @@ impl BlockChainClient for Client { return Some(block.rlp_bytes(Seal::Without)); } } - Self::block_hash(&self.chain, id).and_then(|hash| { - self.chain.block(&hash) + let chain = self.chain.read(); + Self::block_hash(&chain, id).and_then(|hash| { + chain.block(&hash) }) } fn block_status(&self, id: BlockID) -> BlockStatus { - match Self::block_hash(&self.chain, id) { - Some(ref hash) if self.chain.is_known(hash) => BlockStatus::InChain, + let chain = self.chain.read(); + match Self::block_hash(&chain, id) { + Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain, Some(hash) => self.block_queue.block_status(&hash), None => BlockStatus::Unknown } @@ -785,7 +815,8 @@ impl BlockChainClient for Client { return Some(*block.header.difficulty() + self.block_total_difficulty(BlockID::Latest).expect("blocks in chain have details; qed")); } } - Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_details(&hash)).map(|d| d.total_difficulty) + let chain = self.chain.read(); + Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) } fn nonce(&self, address: &Address, id: BlockID) -> Option { @@ -793,7 +824,8 @@ impl BlockChainClient for Client { } fn block_hash(&self, id: BlockID) -> Option { - Self::block_hash(&self.chain, id) + let chain = self.chain.read(); + Self::block_hash(&chain, id) } fn code(&self, address: &Address, id: BlockID) -> Option> { @@ -809,7 +841,7 @@ impl BlockChainClient for Client { } fn transaction(&self, id: TransactionID) -> Option { - self.transaction_address(id).and_then(|address| self.chain.transaction(&address)) + self.transaction_address(id).and_then(|address| self.chain.read().transaction(&address)) } fn uncle(&self, id: UncleID) -> Option { @@ -818,11 +850,12 @@ impl BlockChainClient for Client { } fn transaction_receipt(&self, id: TransactionID) -> Option { - self.transaction_address(id).and_then(|address| self.chain.block_number(&address.block_hash).and_then(|block_number| { - let t = self.chain.block_body(&address.block_hash) + let chain = self.chain.read(); + self.transaction_address(id).and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| { + let t = chain.block_body(&address.block_hash) .and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index)); - match (t, self.chain.transaction_receipt(&address)) { + match (t, chain.transaction_receipt(&address)) { (Some(tx), Some(receipt)) => { let block_hash = tx.block_hash.clone(); let block_number = tx.block_number.clone(); @@ -832,7 +865,7 @@ impl BlockChainClient for Client { 0 => U256::zero(), i => { let prior_address = TransactionAddress { block_hash: address.block_hash, index: i - 1 }; - let prior_receipt = self.chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed"); + let prior_receipt = chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed"); prior_receipt.gas_used } }; @@ -863,28 +896,29 @@ impl BlockChainClient for Client { } fn tree_route(&self, from: &H256, to: &H256) -> Option { - match self.chain.is_known(from) && self.chain.is_known(to) { - true => Some(self.chain.tree_route(from.clone(), to.clone())), + let chain = self.chain.read(); + match chain.is_known(from) && chain.is_known(to) { + true => Some(chain.tree_route(from.clone(), to.clone())), false => None } } fn find_uncles(&self, hash: &H256) -> Option> { - self.chain.find_uncle_hashes(hash, self.engine.maximum_uncle_age()) + self.chain.read().find_uncle_hashes(hash, self.engine.maximum_uncle_age()) } fn state_data(&self, hash: &H256) -> Option { - self.state_db.lock().state(hash) + self.state_db.read().state(hash) } fn block_receipts(&self, hash: &H256) -> Option { - self.chain.block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec()) + self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec()) } fn import_block(&self, bytes: Bytes) -> Result { { let header = BlockView::new(&bytes).header_view(); - if self.chain.is_known(&header.sha3()) { + if self.chain.read().is_known(&header.sha3()) { return Err(BlockImportError::Import(ImportError::AlreadyInChain)); } if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown { @@ -903,12 +937,13 @@ impl BlockChainClient for Client { } fn chain_info(&self) -> BlockChainInfo { + let chain = self.chain.read(); BlockChainInfo { - total_difficulty: self.chain.best_block_total_difficulty(), - pending_total_difficulty: self.chain.best_block_total_difficulty(), - genesis_hash: self.chain.genesis_hash(), - best_block_hash: self.chain.best_block_hash(), - best_block_number: From::from(self.chain.best_block_number()) + total_difficulty: chain.best_block_total_difficulty(), + pending_total_difficulty: chain.best_block_total_difficulty(), + genesis_hash: chain.genesis_hash(), + best_block_hash: chain.best_block_hash(), + best_block_number: From::from(chain.best_block_number()) } } @@ -918,7 +953,7 @@ impl BlockChainClient for Client { fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockID, to_block: BlockID) -> Option> { match (self.block_number(from_block), self.block_number(to_block)) { - (Some(from), Some(to)) => Some(self.chain.blocks_with_bloom(bloom, from, to)), + (Some(from), Some(to)) => Some(self.chain.read().blocks_with_bloom(bloom, from, to)), _ => None } } @@ -936,10 +971,11 @@ impl BlockChainClient for Client { blocks.sort(); + let chain = self.chain.read(); blocks.into_iter() - .filter_map(|number| self.chain.block_hash(number).map(|hash| (number, hash))) - .filter_map(|(number, hash)| self.chain.block_receipts(&hash).map(|r| (number, hash, r.receipts))) - .filter_map(|(number, hash, receipts)| self.chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes()))) + .filter_map(|number| chain.block_hash(number).map(|hash| (number, hash))) + .filter_map(|(number, hash)| chain.block_receipts(&hash).map(|r| (number, hash, r.receipts))) + .filter_map(|(number, hash, receipts)| chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes()))) .flat_map(|(number, hash, receipts, hashes)| { let mut log_index = 0; receipts.into_iter() @@ -975,7 +1011,7 @@ impl BlockChainClient for Client { to_address: From::from(filter.to_address), }; - let traces = self.tracedb.filter(&filter); + let traces = self.tracedb.read().filter(&filter); Some(traces) } else { None @@ -987,7 +1023,7 @@ impl BlockChainClient for Client { self.transaction_address(trace.transaction) .and_then(|tx_address| { self.block_number(BlockID::Hash(tx_address.block_hash)) - .and_then(|number| self.tracedb.trace(number, tx_address.index, trace_address)) + .and_then(|number| self.tracedb.read().trace(number, tx_address.index, trace_address)) }) } @@ -995,17 +1031,17 @@ impl BlockChainClient for Client { self.transaction_address(transaction) .and_then(|tx_address| { self.block_number(BlockID::Hash(tx_address.block_hash)) - .and_then(|number| self.tracedb.transaction_traces(number, tx_address.index)) + .and_then(|number| self.tracedb.read().transaction_traces(number, tx_address.index)) }) } fn block_traces(&self, block: BlockID) -> Option> { self.block_number(block) - .and_then(|number| self.tracedb.block_traces(number)) + .and_then(|number| self.tracedb.read().block_traces(number)) } fn last_hashes(&self) -> LastHashes { - (*self.build_last_hashes(self.chain.best_block_hash())).clone() + (*self.build_last_hashes(self.chain.read().best_block_hash())).clone() } fn queue_transactions(&self, transactions: Vec) { @@ -1032,14 +1068,15 @@ impl BlockChainClient for Client { impl MiningBlockChainClient for Client { fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock { let engine = &*self.engine; - let h = self.chain.best_block_hash(); + let chain = self.chain.read(); + let h = chain.best_block_hash(); let mut open_block = OpenBlock::new( engine, self.factories.clone(), false, // TODO: this will need to be parameterised once we want to do immediate mining insertion. - self.state_db.lock().boxed_clone(), - &self.chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"), + self.state_db.read().boxed_clone(), + &chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"), self.build_last_hashes(h.clone()), author, gas_range_target, @@ -1047,7 +1084,7 @@ impl MiningBlockChainClient for Client { ).expect("OpenBlock::new only fails if parent state root invalid; state root of best block's header is never invalid; qed"); // Add uncles - self.chain + chain .find_uncle_headers(&h, engine.maximum_uncle_age()) .unwrap() .into_iter() @@ -1088,7 +1125,7 @@ impl MiningBlockChainClient for Client { precise_time_ns() - start, ); }); - self.db.flush().expect("DB flush failed."); + self.db.read().flush().expect("DB flush failed."); Ok(h) } } diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 504ca4de7..bb70de6cd 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -62,7 +62,7 @@ impl FromStr for DatabaseCompactionProfile { } /// Operating mode for the client. -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq, Clone)] pub enum Mode { /// Always on. Active, diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index a2533ecde..c9d60f075 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -227,6 +227,11 @@ impl Miner { self.options.force_sealing || !self.options.new_work_notify.is_empty() } + /// Clear all pending block states + pub fn clear(&self) { + self.sealing_work.lock().queue.reset(); + } + /// Get `Some` `clone()` of the current pending block's state or `None` if we're not sealing. pub fn pending_state(&self) -> Option { self.sealing_work.lock().queue.peek_last_ref().map(|b| b.block().fields().state.clone()) diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index e2e4772a4..4c26e2e44 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -78,7 +78,7 @@ impl ClientService { let pruning = config.pruning; let client = try!(Client::new(config, &spec, db_path, miner, io_service.channel())); - let snapshot = try!(SnapshotService::new(spec, pruning, db_path.into(), io_service.channel())); + let snapshot = try!(SnapshotService::new(spec, pruning, db_path.into(), io_service.channel(), client.clone())); let snapshot = Arc::new(snapshot); @@ -90,7 +90,7 @@ impl ClientService { try!(io_service.register_handler(client_io)); let stop_guard = ::devtools::StopGuard::new(); - run_ipc(ipc_path, client.clone(), stop_guard.share()); + run_ipc(ipc_path, client.clone(), snapshot.clone(), stop_guard.share()); Ok(ClientService { io_service: Arc::new(io_service), @@ -176,14 +176,27 @@ impl IoHandler for ClientIoHandler { } #[cfg(feature="ipc")] -fn run_ipc(base_path: &Path, client: Arc, stop: Arc) { +fn run_ipc(base_path: &Path, client: Arc, snapshot_service: Arc, stop: Arc) { let mut path = base_path.to_owned(); path.push("parity-chain.ipc"); let socket_addr = format!("ipc://{}", path.to_string_lossy()); + let s = stop.clone(); ::std::thread::spawn(move || { let mut worker = nanoipc::Worker::new(&(client as Arc)); worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues"); + while !s.load(::std::sync::atomic::Ordering::Relaxed) { + worker.poll(); + } + }); + + let mut path = base_path.to_owned(); + path.push("parity-snapshot.ipc"); + let socket_addr = format!("ipc://{}", path.to_string_lossy()); + ::std::thread::spawn(move || { + let mut worker = nanoipc::Worker::new(&(snapshot_service as Arc<::snapshot::SnapshotService>)); + worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues"); + while !stop.load(::std::sync::atomic::Ordering::Relaxed) { worker.poll(); } @@ -191,7 +204,7 @@ fn run_ipc(base_path: &Path, client: Arc, stop: Arc) { } #[cfg(not(feature="ipc"))] -fn run_ipc(_base_path: &Path, _client: Arc, _stop: Arc) { +fn run_ipc(_base_path: &Path, _client: Arc, _snapshot_service: Arc, _stop: Arc) { } #[cfg(test)] diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 2a81b967d..ab23be9b3 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -32,9 +32,9 @@ use util::Mutex; use util::hash::{FixedHash, H256}; use util::journaldb::{self, Algorithm, JournalDB}; use util::kvdb::Database; -use util::sha3::SHA3_NULL_RLP; use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut}; -use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType}; +use util::sha3::SHA3_NULL_RLP; +use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType}; use self::account::Account; use self::block::AbridgedBlock; @@ -44,7 +44,10 @@ use crossbeam::{scope, ScopedJoinHandle}; use rand::{Rng, OsRng}; pub use self::error::Error; -pub use self::service::{RestorationStatus, Service, SnapshotService}; +pub use self::service::{Service, DatabaseRestore}; +pub use self::traits::{SnapshotService, RemoteSnapshotService}; +pub use types::snapshot_manifest::ManifestData; +pub use types::restoration_status::RestorationStatus; pub mod io; pub mod service; @@ -56,6 +59,11 @@ mod error; #[cfg(test)] mod tests; +mod traits { + #![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues + include!(concat!(env!("OUT_DIR"), "/snapshot_service_trait.rs")); +} + // Try to have chunks be around 4MB (before compression) const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024; @@ -354,54 +362,6 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, - /// List of block chunk hashes. - pub block_hashes: Vec, - /// The final, expected state root. - pub state_root: H256, - /// Block number this snapshot was taken at. - pub block_number: u64, - /// Block hash this snapshot was taken at. - pub block_hash: H256, -} - -impl ManifestData { - /// Encode the manifest data to rlp. - pub fn into_rlp(self) -> Bytes { - let mut stream = RlpStream::new_list(5); - stream.append(&self.state_hashes); - stream.append(&self.block_hashes); - stream.append(&self.state_root); - stream.append(&self.block_number); - stream.append(&self.block_hash); - - stream.out() - } - - /// Try to restore manifest data from raw bytes, interpreted as RLP. - pub fn from_rlp(raw: &[u8]) -> Result { - let decoder = UntrustedRlp::new(raw); - - let state_hashes: Vec = try!(decoder.val_at(0)); - let block_hashes: Vec = try!(decoder.val_at(1)); - let state_root: H256 = try!(decoder.val_at(2)); - let block_number: u64 = try!(decoder.val_at(3)); - let block_hash: H256 = try!(decoder.val_at(4)); - - Ok(ManifestData { - state_hashes: state_hashes, - block_hashes: block_hashes, - state_root: state_root, - block_number: block_number, - block_hash: block_hash, - }) - } -} - /// Used to rebuild the state trie piece by piece. pub struct StateRebuilder { db: Box, @@ -653,4 +613,4 @@ impl BlockRebuilder { } } } -} \ No newline at end of file +} diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index 40f629ad9..f91821b22 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -23,7 +23,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use super::{ManifestData, StateRebuilder, BlockRebuilder}; +use super::{ManifestData, StateRebuilder, BlockRebuilder, RestorationStatus, SnapshotService}; use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}; use blockchain::BlockChain; @@ -39,51 +39,10 @@ use util::journaldb::Algorithm; use util::kvdb::{Database, DatabaseConfig}; use util::snappy; -/// Statuses for restorations. -#[derive(PartialEq, Eq, Clone, Copy, Debug)] -pub enum RestorationStatus { - /// No restoration. - Inactive, - /// Ongoing restoration. - Ongoing, - /// Failed restoration. - Failed, -} - -/// The interface for a snapshot network service. -/// This handles: -/// - restoration of snapshots to temporary databases. -/// - responding to queries for snapshot manifests and chunks -pub trait SnapshotService { - /// Query the most recent manifest data. - fn manifest(&self) -> Option; - - /// Get raw chunk for a given hash. - fn chunk(&self, hash: H256) -> Option; - - /// Ask the snapshot service for the restoration status. - fn status(&self) -> RestorationStatus; - - /// Ask the snapshot service for the number of chunks completed. - /// Return a tuple of (state_chunks, block_chunks). - /// Undefined when not restoring. - fn chunks_done(&self) -> (usize, usize); - - /// Begin snapshot restoration. - /// If restoration in-progress, this will reset it. - /// From this point on, any previous snapshot may become unavailable. - fn begin_restore(&self, manifest: ManifestData); - - /// Abort an in-progress restoration if there is one. - fn abort_restore(&self); - - /// Feed a raw state chunk to the service to be processed asynchronously. - /// no-op if not currently restoring. - fn restore_state_chunk(&self, hash: H256, chunk: Bytes); - - /// Feed a raw block chunk to the service to be processed asynchronously. - /// no-op if currently restoring. - fn restore_block_chunk(&self, hash: H256, chunk: Bytes); +/// External database restoration handler +pub trait DatabaseRestore : Send + Sync { + /// Restart with a new backend. Takes ownership of passed database and moves it to a new location. + fn restore_db(&self, new_db: &str) -> Result<(), Error>; } /// State restoration manager. @@ -208,11 +167,12 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, + db_restore: Arc, } impl Service { /// Create a new snapshot service. - pub fn new(spec: &Spec, pruning: Algorithm, client_db: PathBuf, io_channel: Channel) -> Result { + pub fn new(spec: &Spec, pruning: Algorithm, client_db: PathBuf, io_channel: Channel, db_restore: Arc) -> Result { let db_path = try!(client_db.parent().and_then(Path::parent) .ok_or_else(|| UtilError::SimpleString("Failed to find database root.".into()))).to_owned(); @@ -236,6 +196,7 @@ impl Service { genesis_block: spec.genesis_block(), state_chunks: AtomicUsize::new(0), block_chunks: AtomicUsize::new(0), + db_restore: db_restore, }; // create the root snapshot dir if it doesn't exist. @@ -295,37 +256,8 @@ impl Service { let our_db = self.restoration_db(); trace!(target: "snapshot", "replacing {:?} with {:?}", self.client_db, our_db); - - let mut backup_db = self.restoration_dir(); - backup_db.push("backup_db"); - - let _ = fs::remove_dir_all(&backup_db); - - let existed = match fs::rename(&self.client_db, &backup_db) { - Ok(_) => true, - Err(e) => if let ErrorKind::NotFound = e.kind() { - false - } else { - return Err(e.into()); - } - }; - - match fs::rename(&our_db, &self.client_db) { - Ok(_) => { - // clean up the backup. - if existed { - try!(fs::remove_dir_all(&backup_db)); - } - Ok(()) - } - Err(e) => { - // restore the backup. - if existed { - try!(fs::rename(&backup_db, &self.client_db)); - } - Err(e.into()) - } - } + try!(self.db_restore.restore_db(our_db.to_str().unwrap())); + Ok(()) } /// Initialize the restoration synchronously. @@ -360,7 +292,10 @@ impl Service { *res = Some(try!(Restoration::new(params))); - *self.status.lock() = RestorationStatus::Ongoing; + *self.status.lock() = RestorationStatus::Ongoing { + state_chunks_done: self.state_chunks.load(Ordering::Relaxed) as u32, + block_chunks_done: self.block_chunks.load(Ordering::Relaxed) as u32, + }; Ok(()) } @@ -418,7 +353,7 @@ impl Service { match self.status() { RestorationStatus::Inactive | RestorationStatus::Failed => Ok(()), - RestorationStatus::Ongoing => { + RestorationStatus::Ongoing { .. } => { let res = { let rest = match *restoration { Some(ref mut r) => r, @@ -489,10 +424,6 @@ impl SnapshotService for Service { *self.status.lock() } - fn chunks_done(&self) -> (usize, usize) { - (self.state_chunks.load(Ordering::Relaxed), self.block_chunks.load(Ordering::Relaxed)) - } - fn begin_restore(&self, manifest: ManifestData) { self.io_channel.send(ClientIoMessage::BeginRestoration(manifest)) .expect("snapshot service and io service are kept alive by client service; qed"); @@ -522,15 +453,23 @@ impl SnapshotService for Service { #[cfg(test)] mod tests { + use std::sync::Arc; use service::ClientIoMessage; use io::{IoService}; use devtools::RandomTempPath; use tests::helpers::get_test_spec; use util::journaldb::Algorithm; - - use snapshot::ManifestData; + use error::Error; + use snapshot::{ManifestData, RestorationStatus, SnapshotService}; use super::*; + struct NoopDBRestore; + impl DatabaseRestore for NoopDBRestore { + fn restore_db(&self, _new_db: &str) -> Result<(), Error> { + Ok(()) + } + } + #[test] fn sends_async_messages() { let service = IoService::::start().unwrap(); @@ -544,13 +483,13 @@ mod tests { &get_test_spec(), Algorithm::Archive, dir, - service.channel() + service.channel(), + Arc::new(NoopDBRestore), ).unwrap(); assert!(service.manifest().is_none()); assert!(service.chunk(Default::default()).is_none()); assert_eq!(service.status(), RestorationStatus::Inactive); - assert_eq!(service.chunks_done(), (0, 0)); let manifest = ManifestData { state_hashes: vec![], @@ -565,4 +504,10 @@ mod tests { service.restore_state_chunk(Default::default(), vec![]); service.restore_block_chunk(Default::default(), vec![]); } -} \ No newline at end of file +} + +impl Drop for Service { + fn drop(&mut self) { + self.abort_restore(); + } +} diff --git a/ethcore/src/snapshot/snapshot_service_trait.rs b/ethcore/src/snapshot/snapshot_service_trait.rs new file mode 100644 index 000000000..7df90c943 --- /dev/null +++ b/ethcore/src/snapshot/snapshot_service_trait.rs @@ -0,0 +1,54 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use super::{ManifestData, RestorationStatus}; +use util::{Bytes, H256}; +use ipc::IpcConfig; + +/// The interface for a snapshot network service. +/// This handles: +/// - restoration of snapshots to temporary databases. +/// - responding to queries for snapshot manifests and chunks +#[derive(Ipc)] +#[ipc(client_ident="RemoteSnapshotService")] +pub trait SnapshotService : Sync + Send { + /// Query the most recent manifest data. + fn manifest(&self) -> Option; + + /// Get raw chunk for a given hash. + fn chunk(&self, hash: H256) -> Option; + + /// Ask the snapshot service for the restoration status. + fn status(&self) -> RestorationStatus; + + /// Begin snapshot restoration. + /// If restoration in-progress, this will reset it. + /// From this point on, any previous snapshot may become unavailable. + fn begin_restore(&self, manifest: ManifestData); + + /// Abort an in-progress restoration if there is one. + fn abort_restore(&self); + + /// Feed a raw state chunk to the service to be processed asynchronously. + /// no-op if not currently restoring. + fn restore_state_chunk(&self, hash: H256, chunk: Bytes); + + /// Feed a raw block chunk to the service to be processed asynchronously. + /// no-op if currently restoring. + fn restore_block_chunk(&self, hash: H256, chunk: Bytes); +} + +impl IpcConfig for SnapshotService { } diff --git a/ethcore/src/types/mod.rs.in b/ethcore/src/types/mod.rs.in index e7731d1cc..0537fe056 100644 --- a/ethcore/src/types/mod.rs.in +++ b/ethcore/src/types/mod.rs.in @@ -31,3 +31,5 @@ pub mod trace_filter; pub mod call_analytics; pub mod transaction_import; pub mod block_import_error; +pub mod restoration_status; +pub mod snapshot_manifest; diff --git a/ethcore/src/types/restoration_status.rs b/ethcore/src/types/restoration_status.rs new file mode 100644 index 000000000..2840d9416 --- /dev/null +++ b/ethcore/src/types/restoration_status.rs @@ -0,0 +1,34 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Restoration status type definition + +/// Statuses for restorations. +#[derive(PartialEq, Eq, Clone, Copy, Debug, Binary)] +pub enum RestorationStatus { + /// No restoration. + Inactive, + /// Ongoing restoration. + Ongoing { + /// Number of state chunks completed. + state_chunks_done: u32, + /// Number of block chunks completed. + block_chunks_done: u32, + }, + /// Failed restoration. + Failed, +} + diff --git a/ethcore/src/types/snapshot_manifest.rs b/ethcore/src/types/snapshot_manifest.rs new file mode 100644 index 000000000..859ec016f --- /dev/null +++ b/ethcore/src/types/snapshot_manifest.rs @@ -0,0 +1,70 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Snapshot manifest type definition + +use util::hash::H256; +use rlp::*; +use util::Bytes; + +/// Manifest data. +#[derive(Debug, Clone, PartialEq, Eq, Binary)] +pub struct ManifestData { + /// List of state chunk hashes. + pub state_hashes: Vec, + /// List of block chunk hashes. + pub block_hashes: Vec, + /// The final, expected state root. + pub state_root: H256, + /// Block number this snapshot was taken at. + pub block_number: u64, + /// Block hash this snapshot was taken at. + pub block_hash: H256, +} + +impl ManifestData { + /// Encode the manifest data to rlp. + pub fn into_rlp(self) -> Bytes { + let mut stream = RlpStream::new_list(5); + stream.append(&self.state_hashes); + stream.append(&self.block_hashes); + stream.append(&self.state_root); + stream.append(&self.block_number); + stream.append(&self.block_hash); + + stream.out() + } + + /// Try to restore manifest data from raw bytes, interpreted as RLP. + pub fn from_rlp(raw: &[u8]) -> Result { + let decoder = UntrustedRlp::new(raw); + + let state_hashes: Vec = try!(decoder.val_at(0)); + let block_hashes: Vec = try!(decoder.val_at(1)); + let state_root: H256 = try!(decoder.val_at(2)); + let block_number: u64 = try!(decoder.val_at(3)); + let block_hash: H256 = try!(decoder.val_at(4)); + + Ok(ManifestData { + state_hashes: state_hashes, + block_hashes: block_hashes, + state_root: state_root, + block_number: block_number, + block_hash: block_hash, + }) + } +} + diff --git a/ethcore/src/verification/mod.rs b/ethcore/src/verification/mod.rs index 53c38a6b0..ed9c8ebc7 100644 --- a/ethcore/src/verification/mod.rs +++ b/ethcore/src/verification/mod.rs @@ -25,7 +25,7 @@ pub use self::canon_verifier::CanonVerifier; pub use self::noop_verifier::NoopVerifier; /// Verifier type. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum VerifierType { /// Verifies block normally. Canon, diff --git a/parity/main.rs b/parity/main.rs index 86844baa9..9c2ae7942 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -99,9 +99,11 @@ mod modules; mod account; mod blockchain; mod presale; -mod run; -mod sync; mod snapshot; +mod run; +#[cfg(feature="ipc")] +mod sync; +#[cfg(feature="ipc")] mod boot; #[cfg(feature="stratum")] @@ -158,10 +160,24 @@ mod stratum_optional { } } -fn main() { +#[cfg(not(feature="ipc"))] +fn sync_main() -> bool { + false +} + +#[cfg(feature="ipc")] +fn sync_main() -> bool { // just redirect to the sync::main() if std::env::args().nth(1).map_or(false, |arg| arg == "sync") { sync::main(); + true + } else { + false + } +} + +fn main() { + if sync_main() { return; } diff --git a/parity/modules.rs b/parity/modules.rs index 5edbca702..73de6ca29 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use ethcore::client::BlockChainClient; use hypervisor::Hypervisor; use ethsync::{SyncConfig, NetworkConfiguration, NetworkError}; +use ethcore::snapshot::SnapshotService; #[cfg(not(feature="ipc"))] use self::no_ipc_deps::*; #[cfg(feature="ipc")] @@ -25,10 +26,12 @@ use self::ipc_deps::*; use ethcore_logger::Config as LogConfig; use std::path::Path; +#[cfg(feature="ipc")] pub mod service_urls { use std::path::PathBuf; pub const CLIENT: &'static str = "parity-chain.ipc"; + pub const SNAPSHOT: &'static str = "parity-snapshot.ipc"; pub const SYNC: &'static str = "parity-sync.ipc"; pub const SYNC_NOTIFY: &'static str = "parity-sync-notify.ipc"; pub const NETWORK_MANAGER: &'static str = "parity-manage-net.ipc"; @@ -119,6 +122,7 @@ pub fn sync sync_cfg: SyncConfig, net_cfg: NetworkConfiguration, _client: Arc, + _snapshot_service: Arc, log_settings: &LogConfig, ) -> Result @@ -148,10 +152,11 @@ pub fn sync sync_cfg: SyncConfig, net_cfg: NetworkConfiguration, client: Arc, + snapshot_service: Arc, _log_settings: &LogConfig, ) -> Result { - let eth_sync = try!(EthSync::new(sync_cfg, client, net_cfg)); + let eth_sync = try!(EthSync::new(sync_cfg, client, snapshot_service, net_cfg)); Ok((eth_sync.clone() as Arc, eth_sync.clone() as Arc, eth_sync.clone() as Arc)) } diff --git a/parity/run.rs b/parity/run.rs index 8a68fe1af..174fe315f 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -179,13 +179,14 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> { // take handle to client let client = service.client(); + let snapshot_service = service.snapshot_service(); // create external miner let external_miner = Arc::new(ExternalMiner::default()); // create sync object let (sync_provider, manage_network, chain_notify) = try!(modules::sync( - &mut hypervisor, sync_config, net_conf.into(), client.clone(), &cmd.logger_config, + &mut hypervisor, sync_config, net_conf.into(), client.clone(), snapshot_service, &cmd.logger_config, ).map_err(|e| format!("Sync error: {}", e))); service.add_notify(chain_notify.clone()); diff --git a/parity/snapshot.rs b/parity/snapshot.rs index ecc463a2e..5bf5024ae 100644 --- a/parity/snapshot.rs +++ b/parity/snapshot.rs @@ -129,10 +129,9 @@ impl SnapshotCommand { let informant_handle = snapshot.clone(); ::std::thread::spawn(move || { - while let RestorationStatus::Ongoing = informant_handle.status() { - let (state_chunks, block_chunks) = informant_handle.chunks_done(); + while let RestorationStatus::Ongoing { state_chunks_done, block_chunks_done } = informant_handle.status() { info!("Processed {}/{} state chunks and {}/{} block chunks.", - state_chunks, num_state, block_chunks, num_blocks); + state_chunks_done, num_state, block_chunks_done, num_blocks); ::std::thread::sleep(Duration::from_secs(5)); } @@ -161,7 +160,7 @@ impl SnapshotCommand { } match snapshot.status() { - RestorationStatus::Ongoing => Err("Snapshot file is incomplete and missing chunks.".into()), + RestorationStatus::Ongoing { .. } => Err("Snapshot file is incomplete and missing chunks.".into()), RestorationStatus::Failed => Err("Snapshot restoration failed.".into()), RestorationStatus::Inactive => { info!("Restoration complete."); diff --git a/parity/sync.rs b/parity/sync.rs index 27e9d5a6a..85f771546 100644 --- a/parity/sync.rs +++ b/parity/sync.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use hypervisor::{SYNC_MODULE_ID, HYPERVISOR_IPC_URL, ControlService}; use ethcore::client::{RemoteClient, ChainNotify}; +use ethcore::snapshot::{RemoteSnapshotService}; use ethsync::{SyncProvider, EthSync, ManageNetwork, ServiceConfiguration}; use modules::service_urls; use boot; @@ -45,8 +46,9 @@ pub fn main() { .unwrap_or_else(|e| panic!("Fatal: error reading boot arguments ({:?})", e)); let remote_client = dependency!(RemoteClient, &service_urls::with_base(&service_config.io_path, service_urls::CLIENT)); + let remote_snapshot = dependency!(RemoteSnapshotService, &service_urls::with_base(&service_config.io_path, service_urls::SNAPSHOT)); - let sync = EthSync::new(service_config.sync, remote_client.service().clone(), service_config.net).unwrap(); + let sync = EthSync::new(service_config.sync, remote_client.service().clone(), remote_snapshot.service().clone(), service_config.net).unwrap(); let _ = boot::main_thread(); let service_stop = Arc::new(AtomicBool::new(false)); diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 7807c01eb..9487f020d 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -254,7 +254,8 @@ impl Eth for EthClient where let status = take_weak!(self.sync).status(); let res = match status.state { SyncState::Idle => SyncStatus::None, - SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead => { + SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead + | SyncState::SnapshotManifest | SyncState::SnapshotData | SyncState::SnapshotWaiting => { let current_block = U256::from(take_weak!(self.client).chain_info().best_block_number); let highest_block = U256::from(status.highest_block_number.unwrap_or(status.start_block_number)); diff --git a/rpc/src/v1/tests/helpers/sync_provider.rs b/rpc/src/v1/tests/helpers/sync_provider.rs index 94f7b4893..b83aff758 100644 --- a/rpc/src/v1/tests/helpers/sync_provider.rs +++ b/rpc/src/v1/tests/helpers/sync_provider.rs @@ -49,6 +49,8 @@ impl TestSyncProvider { num_peers: config.num_peers, num_active_peers: 0, mem_used: 0, + num_snapshot_chunks: 0, + snapshot_chunks_done: 0, }), } } diff --git a/sync/src/api.rs b/sync/src/api.rs index a19004642..de1769d9c 100644 --- a/sync/src/api.rs +++ b/sync/src/api.rs @@ -20,6 +20,7 @@ use network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId, use util::{U256, H256}; use io::{TimerToken}; use ethcore::client::{BlockChainClient, ChainNotify}; +use ethcore::snapshot::SnapshotService; use ethcore::header::BlockNumber; use sync_io::NetSyncIo; use chain::{ChainSync, SyncStatus}; @@ -71,12 +72,12 @@ pub struct EthSync { impl EthSync { /// Creates and register protocol with the network service - pub fn new(config: SyncConfig, chain: Arc, network_config: NetworkConfiguration) -> Result, NetworkError> { + pub fn new(config: SyncConfig, chain: Arc, snapshot_service: Arc, network_config: NetworkConfiguration) -> Result, NetworkError> { let chain_sync = ChainSync::new(config, &*chain); let service = try!(NetworkService::new(try!(network_config.into_basic()))); let sync = Arc::new(EthSync{ network: service, - handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain }), + handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain, snapshot_service: snapshot_service }), }); Ok(sync) @@ -93,8 +94,10 @@ impl SyncProvider for EthSync { } struct SyncProtocolHandler { - /// Shared blockchain client. TODO: this should evetually become an IPC endpoint + /// Shared blockchain client. chain: Arc, + /// Shared snapshot service. + snapshot_service: Arc, /// Sync strategy sync: RwLock, } @@ -105,21 +108,21 @@ impl NetworkProtocolHandler for SyncProtocolHandler { } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { - ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain), *peer, packet_id, data); + ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer, packet_id, data); } fn connected(&self, io: &NetworkContext, peer: &PeerId) { - self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain), *peer); + self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer); } fn disconnected(&self, io: &NetworkContext, peer: &PeerId) { - self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain), *peer); + self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer); } fn timeout(&self, io: &NetworkContext, _timer: TimerToken) { - self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain)); - self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain)); - self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain)); + self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service)); + self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service)); + self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service)); } } @@ -133,7 +136,7 @@ impl ChainNotify for EthSync { _duration: u64) { self.network.with_context(ETH_PROTOCOL, |context| { - let mut sync_io = NetSyncIo::new(context, &*self.handler.chain); + let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service); self.handler.sync.write().chain_new_blocks( &mut sync_io, &imported, @@ -146,7 +149,7 @@ impl ChainNotify for EthSync { fn start(&self) { self.network.start().unwrap_or_else(|e| warn!("Error starting network: {:?}", e)); - self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8]) + self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8, 64u8]) .unwrap_or_else(|e| warn!("Error registering ethereum protocol: {:?}", e)); } @@ -202,7 +205,7 @@ impl ManageNetwork for EthSync { fn stop_network(&self) { self.network.with_context(ETH_PROTOCOL, |context| { - let mut sync_io = NetSyncIo::new(context, &*self.handler.chain); + let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service); self.handler.sync.write().abort(&mut sync_io); }); self.stop(); diff --git a/sync/src/chain.rs b/sync/src/chain.rs index e5e5de5dc..ea5e593f3 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -96,17 +96,19 @@ use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::client::{BlockChainClient, BlockStatus, BlockID, BlockChainInfo, BlockImportError}; use ethcore::error::*; use ethcore::block::Block; +use ethcore::snapshot::{ManifestData, RestorationStatus}; use sync_io::SyncIo; use time; use super::SyncConfig; use blocks::BlockCollection; +use snapshot::{Snapshot, ChunkType}; use rand::{thread_rng, Rng}; known_heap_size!(0, PeerInfo); type PacketDecodeError = DecoderError; -const PROTOCOL_VERSION: u8 = 63u8; +const PROTOCOL_VERSION: u8 = 64u8; const MAX_BODIES_TO_SEND: usize = 256; const MAX_HEADERS_TO_SEND: usize = 512; const MAX_NODE_DATA_TO_SEND: usize = 1024; @@ -136,14 +138,26 @@ const GET_NODE_DATA_PACKET: u8 = 0x0d; const NODE_DATA_PACKET: u8 = 0x0e; const GET_RECEIPTS_PACKET: u8 = 0x0f; const RECEIPTS_PACKET: u8 = 0x10; +const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11; +const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12; +const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13; +const SNAPSHOT_DATA_PACKET: u8 = 0x14; const HEADERS_TIMEOUT_SEC: f64 = 15f64; const BODIES_TIMEOUT_SEC: f64 = 5f64; const FORK_HEADER_TIMEOUT_SEC: f64 = 3f64; +const SNAPSHOT_MANIFEST_TIMEOUT_SEC: f64 = 3f64; +const SNAPSHOT_DATA_TIMEOUT_SEC: f64 = 10f64; #[derive(Copy, Clone, Eq, PartialEq, Debug)] /// Sync state pub enum SyncState { + /// Waiting for pv64 peers to start snapshot syncing + SnapshotManifest, + /// Downloading snapshot data + SnapshotData, + /// Waiting for snapshot restoration to complete + SnapshotWaiting, /// Downloading subchain heads ChainHead, /// Initial chain sync complete. Waiting for new packets @@ -177,10 +191,14 @@ pub struct SyncStatus { pub blocks_received: BlockNumber, /// Total number of connected peers pub num_peers: usize, - /// Total number of active peers + /// Total number of active peers. pub num_active_peers: usize, - /// Heap memory used in bytes + /// Heap memory used in bytes. pub mem_used: usize, + /// Snapshot chunks + pub num_snapshot_chunks: usize, + /// Snapshot chunks downloaded + pub snapshot_chunks_done: usize, } impl SyncStatus { @@ -207,6 +225,8 @@ enum PeerAsking { BlockHeaders, BlockBodies, Heads, + SnapshotManifest, + SnapshotData, } #[derive(Clone, Eq, PartialEq)] @@ -240,6 +260,8 @@ struct PeerInfo { asking_blocks: Vec, /// Holds requested header hash if currently requesting block header by hash asking_hash: Option, + /// Holds requested snapshot chunk hash if any. + asking_snapshot_data: Option, /// Request timestamp ask_time: f64, /// Holds a set of transactions recently sent to this peer to avoid spamming. @@ -248,6 +270,10 @@ struct PeerInfo { expired: bool, /// Peer fork confirmation status confirmation: ForkConfirmation, + /// Best snapshot hash + snapshot_hash: Option, + /// Best snapshot block number + snapshot_number: Option, } impl PeerInfo { @@ -293,6 +319,8 @@ pub struct ChainSync { network_id: U256, /// Optional fork block to check fork_block: Option<(BlockNumber, H256)>, + /// Snapshot downloader. + snapshot: Snapshot, } type RlpResponseResult = Result, PacketDecodeError>; @@ -301,8 +329,8 @@ impl ChainSync { /// Create a new instance of syncing strategy. pub fn new(config: SyncConfig, chain: &BlockChainClient) -> ChainSync { let chain = chain.chain_info(); - let mut sync = ChainSync { - state: SyncState::ChainHead, + ChainSync { + state: SyncState::Idle, starting_block: chain.best_block_number, highest_block: None, last_imported_block: chain.best_block_number, @@ -317,16 +345,15 @@ impl ChainSync { _max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks), network_id: config.network_id, fork_block: config.fork_block, - }; - sync.reset(); - sync + snapshot: Snapshot::new(), + } } /// @returns Synchonization status pub fn status(&self) -> SyncStatus { SyncStatus { state: self.state.clone(), - protocol_version: 63, + protocol_version: if self.state == SyncState::SnapshotData { 64 } else { 63 }, network_id: self.network_id, start_block_number: self.starting_block, last_imported_block_number: Some(self.last_imported_block), @@ -335,6 +362,8 @@ impl ChainSync { blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 }, num_peers: self.peers.values().filter(|p| p.is_allowed()).count(), num_active_peers: self.peers.values().filter(|p| p.is_allowed() && p.asking != PeerAsking::Nothing).count(), + num_snapshot_chunks: self.snapshot.total_chunks(), + snapshot_chunks_done: self.snapshot.done_chunks(), mem_used: self.blocks.heap_size() + self.peers.heap_size_of_children() @@ -350,8 +379,13 @@ impl ChainSync { #[cfg_attr(feature="dev", allow(for_kv_map))] // Because it's not possible to get `values_mut()` /// Reset sync. Clear all downloaded data but keep the queue - fn reset(&mut self) { + fn reset(&mut self, io: &mut SyncIo) { self.blocks.clear(); + self.snapshot.clear(); + if self.state == SyncState::SnapshotData { + debug!(target:"sync", "Aborting snapshot restore"); + io.snapshot_service().abort_restore(); + } for (_, ref mut p) in &mut self.peers { p.asking_blocks.clear(); p.asking_hash = None; @@ -368,7 +402,7 @@ impl ChainSync { /// Restart sync pub fn restart(&mut self, io: &mut SyncIo) { trace!(target: "sync", "Restarting"); - self.reset(); + self.reset(io); self.start_sync_round(io); self.continue_sync(io); } @@ -380,13 +414,19 @@ impl ChainSync { if self.active_peers.is_empty() { trace!(target: "sync", "No more active peers"); if self.state == SyncState::ChainHead { - self.complete_sync(); + self.complete_sync(io); } else { self.restart(io); } } } + fn start_snapshot_sync(&mut self, io: &mut SyncIo, peer_id: PeerId) { + self.snapshot.clear(); + self.request_snapshot_manifest(io, peer_id); + self.state = SyncState::SnapshotManifest; + } + /// Restart sync after bad block has been detected. May end up re-downloading up to QUEUE_SIZE blocks fn restart_on_bad_block(&mut self, io: &mut SyncIo) { // Do not assume that the block queue/chain still has our last_imported_block @@ -398,8 +438,9 @@ impl ChainSync { /// Called by peer to report status fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + let protocol_version: u32 = try!(r.val_at(0)); let peer = PeerInfo { - protocol_version: try!(r.val_at(0)), + protocol_version: protocol_version, network_id: try!(r.val_at(1)), difficulty: Some(try!(r.val_at(2))), latest_hash: try!(r.val_at(3)), @@ -412,6 +453,9 @@ impl ChainSync { last_sent_transactions: HashSet::new(), expired: false, confirmation: if self.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, + asking_snapshot_data: None, + snapshot_hash: if protocol_version == 64 { Some(try!(r.val_at(5))) } else { None }, + snapshot_number: if protocol_version == 64 { Some(try!(r.val_at(6))) } else { None }, }; trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis); @@ -749,6 +793,96 @@ impl ChainSync { Ok(()) } + /// Called when snapshot manifest is downloaded from a peer. + fn on_snapshot_manifest(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id); + return Ok(()); + } + self.clear_peer_download(peer_id); + if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || self.state != SyncState::SnapshotManifest { + trace!(target: "sync", "{}: Ignored unexpected manifest", peer_id); + self.continue_sync(io); + return Ok(()); + } + + let manifest_rlp = try!(r.at(0)); + let manifest = match ManifestData::from_rlp(&manifest_rlp.as_raw()) { + Err(e) => { + trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e); + io.disconnect_peer(peer_id); + self.continue_sync(io); + return Ok(()); + } + Ok(manifest) => manifest, + }; + self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3()); + io.snapshot_service().begin_restore(manifest); + self.state = SyncState::SnapshotData; + + // give a task to the same peer first. + self.sync_peer(io, peer_id, false); + // give tasks to other peers + self.continue_sync(io); + Ok(()) + } + + /// Called when snapshot data is downloaded from a peer. + fn on_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { + if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id); + return Ok(()); + } + self.clear_peer_download(peer_id); + if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || self.state != SyncState::SnapshotData { + trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id); + self.continue_sync(io); + return Ok(()); + } + + // check service status + match io.snapshot_service().status() { + RestorationStatus::Inactive | RestorationStatus::Failed => { + trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id); + self.state = SyncState::Idle; + self.snapshot.clear(); + self.continue_sync(io); + return Ok(()); + }, + RestorationStatus::Ongoing { .. } => { + trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id); + }, + } + + let snapshot_data: Bytes = try!(r.val_at(0)); + match self.snapshot.validate_chunk(&snapshot_data) { + Ok(ChunkType::Block(hash)) => { + trace!(target: "sync", "{}: Processing block chunk", peer_id); + io.snapshot_service().restore_block_chunk(hash, snapshot_data); + } + Ok(ChunkType::State(hash)) => { + trace!(target: "sync", "{}: Processing state chunk", peer_id); + io.snapshot_service().restore_state_chunk(hash, snapshot_data); + } + Err(()) => { + trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id); + io.disconnect_peer(peer_id); + self.continue_sync(io); + return Ok(()); + } + } + + if self.snapshot.is_complete() { + // wait for snapshot restoration process to complete + self.state = SyncState::SnapshotWaiting; + } + // give a task to the same peer first. + self.sync_peer(io, peer_id, false); + // give tasks to other peers + self.continue_sync(io); + Ok(()) + } + /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { trace!(target: "sync", "== Disconnecting {}: {}", peer, io.peer_info(peer)); @@ -764,7 +898,7 @@ impl ChainSync { /// Called when a new peer is connected pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) { trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer)); - if let Err(e) = self.send_status(io) { + if let Err(e) = self.send_status(io, peer) { debug!(target:"sync", "Error sending status request: {:?}", e); io.disable_peer(peer); } @@ -772,24 +906,27 @@ impl ChainSync { /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { - let mut peers: Vec<(PeerId, U256)> = self.peers.iter().filter_map(|(k, p)| - if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero))) } else { None }).collect(); + let mut peers: Vec<(PeerId, U256, u32)> = self.peers.iter().filter_map(|(k, p)| + if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero), p.protocol_version)) } else { None }).collect(); thread_rng().shuffle(&mut peers); //TODO: sort by rating + // prefer peers with higher protocol version + peers.sort_by(|&(_, _, ref v1), &(_, _, ref v2)| v1.cmp(v2)); trace!(target: "sync", "Syncing with {}/{} peers", self.active_peers.len(), peers.len()); - for (p, _) in peers { + for (p, _, _) in peers { if self.active_peers.contains(&p) { self.sync_peer(io, p, false); } } - if self.state != SyncState::Waiting && !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) { - self.complete_sync(); + if self.state != SyncState::Waiting && self.state != SyncState::SnapshotWaiting + && !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) { + self.complete_sync(io); } } /// Called after all blocks have been downloaded - fn complete_sync(&mut self) { + fn complete_sync(&mut self, io: &mut SyncIo) { trace!(target: "sync", "Sync complete"); - self.reset(); + self.reset(io); self.state = SyncState::Idle; } @@ -805,7 +942,7 @@ impl ChainSync { trace!(target: "sync", "Skipping deactivated peer"); return; } - let (peer_latest, peer_difficulty) = { + let (peer_latest, peer_difficulty, peer_snapshot_number, peer_snapshot_hash) = { let peer = self.peers.get_mut(&peer_id).unwrap(); if peer.asking != PeerAsking::Nothing || !peer.can_sync() { return; @@ -814,7 +951,11 @@ impl ChainSync { trace!(target: "sync", "Waiting for the block queue"); return; } - (peer.latest_hash.clone(), peer.difficulty.clone()) + if self.state == SyncState::SnapshotWaiting { + trace!(target: "sync", "Waiting for the snapshot restoration"); + return; + } + (peer.latest_hash.clone(), peer.difficulty.clone(), peer.snapshot_number.as_ref().cloned(), peer.snapshot_hash.as_ref().cloned()) }; let chain_info = io.chain().chain_info(); let td = chain_info.pending_total_difficulty; @@ -823,13 +964,18 @@ impl ChainSync { if force || self.state == SyncState::NewBlocks || peer_difficulty.map_or(true, |pd| pd > syncing_difficulty) { match self.state { SyncState::Idle => { - if self.last_imported_block < chain_info.best_block_number { - self.last_imported_block = chain_info.best_block_number; - self.last_imported_hash = chain_info.best_block_hash; + // check if we can start snapshot sync with this peer + if peer_snapshot_number.unwrap_or(0) > 0 && chain_info.best_block_number == 0 { + self.start_snapshot_sync(io, peer_id); + } else { + if self.last_imported_block < chain_info.best_block_number { + self.last_imported_block = chain_info.best_block_number; + self.last_imported_hash = chain_info.best_block_hash; + } + trace!(target: "sync", "Starting sync with {}", peer_id); + self.start_sync_round(io); + self.sync_peer(io, peer_id, force); } - trace!(target: "sync", "Starting sync with {}", peer_id); - self.start_sync_round(io); - self.sync_peer(io, peer_id, force); }, SyncState::ChainHead => { // Request subchain headers @@ -843,8 +989,14 @@ impl ChainSync { if io.chain().block_status(BlockID::Hash(peer_latest)) == BlockStatus::Unknown { self.request_blocks(io, peer_id, false); } - } - SyncState::Waiting => () + }, + SyncState::SnapshotData => { + if peer_snapshot_hash.is_some() && peer_snapshot_hash == self.snapshot.snapshot_hash() { + self.request_snapshot_data(io, peer_id); + } + }, + SyncState::SnapshotManifest => (), //already downloading from other peer + SyncState::Waiting | SyncState::SnapshotWaiting => () } } } @@ -903,6 +1055,16 @@ impl ChainSync { } } + /// Find some headers or blocks to download for a peer. + fn request_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId) { + self.clear_peer_download(peer_id); + // find chunk data to download + if let Some(hash) = self.snapshot.needed_chunk() { + self.peers.get_mut(&peer_id).unwrap().asking_snapshot_data = Some(hash.clone()); + self.request_snapshot_chunk(io, peer_id, &hash); + } + } + /// Clear all blocks/headers marked as being downloaded by a peer. fn clear_peer_download(&mut self, peer_id: PeerId) { let peer = self.peers.get_mut(&peer_id).unwrap(); @@ -917,9 +1079,15 @@ impl ChainSync { self.blocks.clear_body_download(b); } }, + PeerAsking::SnapshotData => { + if let Some(hash) = peer.asking_snapshot_data { + self.snapshot.clear_chunk_download(&hash); + } + }, _ => (), } peer.asking_blocks.clear(); + peer.asking_snapshot_data = None; } fn block_imported(&mut self, hash: &H256, number: BlockNumber, parent: &H256) { @@ -1016,6 +1184,22 @@ impl ChainSync { rlp.append(&if reverse {1u32} else {0u32}); self.send_request(sync, peer_id, asking, GET_BLOCK_HEADERS_PACKET, rlp.out()); } + + /// Request snapshot manifest from a peer. + fn request_snapshot_manifest(&mut self, sync: &mut SyncIo, peer_id: PeerId) { + trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id); + let rlp = RlpStream::new_list(0); + self.send_request(sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out()); + } + + /// Request snapshot chunk from a peer. + fn request_snapshot_chunk(&mut self, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) { + trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk); + let mut rlp = RlpStream::new_list(1); + rlp.append(chunk); + self.send_request(sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out()); + } + /// Request block bodies from a peer fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec) { let mut rlp = RlpStream::new_list(hashes.len()); @@ -1086,14 +1270,22 @@ impl ChainSync { } /// Send Status message - fn send_status(&mut self, io: &mut SyncIo) -> Result<(), NetworkError> { - let mut packet = RlpStream::new_list(5); + fn send_status(&mut self, io: &mut SyncIo, peer: PeerId) -> Result<(), NetworkError> { + let pv64 = io.eth_protocol_version(peer) >= 64; + let mut packet = RlpStream::new_list(if pv64 { 7 } else { 5 }); let chain = io.chain().chain_info(); packet.append(&(PROTOCOL_VERSION as u32)); packet.append(&self.network_id); packet.append(&chain.total_difficulty); packet.append(&chain.best_block_hash); packet.append(&chain.genesis_hash); + if pv64 { + let manifest = io.snapshot_service().manifest(); + let block_number = manifest.as_ref().map_or(0, |m| m.block_number); + let manifest_hash = manifest.map_or(H256::new(), |m| m.into_rlp().sha3()); + packet.append(&manifest_hash); + packet.append(&block_number); + } io.respond(STATUS_PACKET, packet.out()) } @@ -1230,6 +1422,48 @@ impl ChainSync { Ok(Some((RECEIPTS_PACKET, rlp_result))) } + /// Respond to GetSnapshotManifest request + fn return_snapshot_manifest(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult { + let count = r.item_count(); + trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id); + if count != 0 { + debug!(target: "sync", "Invalid GetSnapshotManifest request, ignoring."); + return Ok(None); + } + let rlp = match io.snapshot_service().manifest() { + Some(manifest) => { + trace!(target: "sync", "{} <- SnapshotManifest", peer_id); + let mut rlp = RlpStream::new_list(1); + rlp.append_raw(&manifest.into_rlp(), 1); + rlp + }, + None => { + trace!(target: "sync", "{}: No manifest to return", peer_id); + let rlp = RlpStream::new_list(0); + rlp + } + }; + Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp))) + } + + /// Respond to GetSnapshotManifest request + fn return_snapshot_data(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult { + let hash: H256 = try!(r.val_at(0)); + trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash); + let rlp = match io.snapshot_service().chunk(hash) { + Some(data) => { + let mut rlp = RlpStream::new_list(1); + rlp.append(&data); + rlp + }, + None => { + let rlp = RlpStream::new_list(0); + rlp + } + }; + Ok(Some((SNAPSHOT_DATA_PACKET, rlp))) + } + fn return_rlp(io: &mut SyncIo, rlp: &UntrustedRlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError> where FRlp : Fn(&SyncIo, &UntrustedRlp, PeerId) -> RlpResponseResult, FError : FnOnce(NetworkError) -> String @@ -1266,6 +1500,14 @@ impl ChainSync { ChainSync::return_node_data, |e| format!("Error sending nodes: {:?}", e)), + GET_SNAPSHOT_MANIFEST_PACKET => ChainSync::return_rlp(io, &rlp, peer, + ChainSync::return_snapshot_manifest, + |e| format!("Error sending snapshot manifest: {:?}", e)), + + GET_SNAPSHOT_DATA_PACKET => ChainSync::return_rlp(io, &rlp, peer, + ChainSync::return_snapshot_data, + |e| format!("Error sending snapshot data: {:?}", e)), + _ => { sync.write().on_packet(io, peer, packet_id, data); Ok(()) @@ -1289,6 +1531,8 @@ impl ChainSync { BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp), NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp), NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), + SNAPSHOT_MANIFEST_PACKET => self.on_snapshot_manifest(io, peer, &rlp), + SNAPSHOT_DATA_PACKET => self.on_snapshot_data(io, peer, &rlp), _ => { debug!(target: "sync", "Unknown packet {}", packet_id); Ok(()) @@ -1308,6 +1552,8 @@ impl ChainSync { PeerAsking::BlockBodies => (tick - peer.ask_time) > BODIES_TIMEOUT_SEC, PeerAsking::Nothing => false, PeerAsking::ForkHeader => (tick - peer.ask_time) > FORK_HEADER_TIMEOUT_SEC, + PeerAsking::SnapshotManifest => (tick - peer.ask_time) > SNAPSHOT_MANIFEST_TIMEOUT_SEC, + PeerAsking::SnapshotData => (tick - peer.ask_time) > SNAPSHOT_DATA_TIMEOUT_SEC, }; if timeout { trace!(target:"sync", "Timeout {}", peer_id); @@ -1321,9 +1567,12 @@ impl ChainSync { } fn check_resume(&mut self, io: &mut SyncIo) { - if !io.chain().queue_info().is_full() && self.state == SyncState::Waiting { + if self.state == SyncState::Waiting && !io.chain().queue_info().is_full() && self.state == SyncState::Waiting { self.state = SyncState::Blocks; self.continue_sync(io); + } else if self.state == SyncState::SnapshotWaiting && io.snapshot_service().status() == RestorationStatus::Inactive { + self.state = SyncState::Idle; + self.continue_sync(io); } } @@ -1559,6 +1808,7 @@ impl ChainSync { #[cfg(test)] mod tests { use tests::helpers::*; + use tests::snapshot::TestSnapshotService; use super::*; use ::SyncConfig; use util::*; @@ -1612,7 +1862,8 @@ mod tests { fn return_receipts_empty() { let mut client = TestBlockChainClient::new(); let mut queue = VecDeque::new(); - let io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let io = TestIo::new(&mut client, &mut ss, &mut queue, None); let result = ChainSync::return_receipts(&io, &UntrustedRlp::new(&[0xc0]), 0); @@ -1624,7 +1875,8 @@ mod tests { let mut client = TestBlockChainClient::new(); let mut queue = VecDeque::new(); let sync = dummy_sync_with_peer(H256::new(), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let mut receipt_list = RlpStream::new_list(4); receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); @@ -1679,7 +1931,8 @@ mod tests { let hashes: Vec<_> = headers.iter().map(|h| HeaderView::new(h).sha3()).collect(); let mut queue = VecDeque::new(); - let io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let io = TestIo::new(&mut client, &mut ss, &mut queue, None); let unknown: H256 = H256::new(); let result = ChainSync::return_block_headers(&io, &UntrustedRlp::new(&make_hash_req(&unknown, 1, 0, false)), 0); @@ -1717,7 +1970,8 @@ mod tests { let mut client = TestBlockChainClient::new(); let mut queue = VecDeque::new(); let sync = dummy_sync_with_peer(H256::new(), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let mut node_list = RlpStream::new_list(3); node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); @@ -1758,6 +2012,9 @@ mod tests { last_sent_transactions: HashSet::new(), expired: false, confirmation: super::ForkConfirmation::Confirmed, + snapshot_number: None, + snapshot_hash: None, + asking_snapshot_data: None, }); sync } @@ -1769,7 +2026,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); let chain_info = client.chain_info(); - let io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let io = TestIo::new(&mut client, &mut ss, &mut queue, None); let lagging_peers = sync.get_lagging_peers(&chain_info, &io); @@ -1800,7 +2058,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peers = sync.get_lagging_peers(&chain_info, &io); let peer_count = sync.propagate_new_hashes(&chain_info, &mut io, &peers); @@ -1820,7 +2079,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peers = sync.get_lagging_peers(&chain_info, &io); let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[], &peers); @@ -1840,7 +2100,8 @@ mod tests { let hash = client.block_hash(BlockID::Number(99)).unwrap(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peers = sync.get_lagging_peers(&chain_info, &io); let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[hash.clone()], &peers); @@ -1859,7 +2120,8 @@ mod tests { client.insert_transaction_to_queue(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); let mut queue = VecDeque::new(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peer_count = sync.propagate_new_transactions(&mut io); // Try to propagate same transactions for the second time let peer_count2 = sync.propagate_new_transactions(&mut io); @@ -1880,7 +2142,8 @@ mod tests { client.insert_transaction_to_queue(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); let mut queue = VecDeque::new(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peer_count = sync.propagate_new_transactions(&mut io); sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[]); // Try to propagate same transactions for the second time @@ -1903,17 +2166,17 @@ mod tests { client.insert_transaction_to_queue(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); let mut queue = VecDeque::new(); + let mut ss = TestSnapshotService::new(); // should sent some { - - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peer_count = sync.propagate_new_transactions(&mut io); assert_eq!(1, io.queue.len()); assert_eq!(1, peer_count); } // Insert some more client.insert_transaction_to_queue(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); // Propagate new transactions let peer_count2 = sync.propagate_new_transactions(&mut io); // And now the peer should have all transactions @@ -1939,7 +2202,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); //sync.have_common_block = true; - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let block = UntrustedRlp::new(&block_data); @@ -1957,7 +2221,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let block = UntrustedRlp::new(&block_data); @@ -1972,7 +2237,8 @@ mod tests { client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let empty_data = vec![]; let block = UntrustedRlp::new(&empty_data); @@ -1988,7 +2254,8 @@ mod tests { client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let hashes_data = get_dummy_hashes(); let hashes_rlp = UntrustedRlp::new(&hashes_data); @@ -2004,7 +2271,8 @@ mod tests { client.add_blocks(10, EachBlockWith::Uncle); let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let empty_hashes_data = vec![]; let hashes_rlp = UntrustedRlp::new(&empty_hashes_data); @@ -2023,7 +2291,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peers = sync.get_lagging_peers(&chain_info, &io); sync.propagate_new_hashes(&chain_info, &mut io, &peers); @@ -2042,7 +2311,8 @@ mod tests { let mut queue = VecDeque::new(); let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); let peers = sync.get_lagging_peers(&chain_info, &io); sync.propagate_blocks(&chain_info, &mut io, &[], &peers); @@ -2076,7 +2346,8 @@ mod tests { // when { let mut queue = VecDeque::new(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks); sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]); assert_eq!(io.chain.miner.status().transactions_in_future_queue, 0); @@ -2090,7 +2361,8 @@ mod tests { } { let mut queue = VecDeque::new(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); io.chain.miner.chain_new_blocks(io.chain, &[], &[], &good_blocks, &retracted_blocks); sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks, &[]); } @@ -2114,7 +2386,8 @@ mod tests { let retracted_blocks = vec![client.block_hash_delta_minus(1)]; let mut queue = VecDeque::new(); - let mut io = TestIo::new(&mut client, &mut queue, None); + let mut ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None); // when sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index c0c240f9d..d2c6e2583 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -26,40 +26,6 @@ //! Implements ethereum protocol version 63 as specified here: //! https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol //! -//! Usage example: -//! -//! ```rust -//! extern crate ethcore_util as util; -//! extern crate ethcore_io as io; -//! extern crate ethcore; -//! extern crate ethsync; -//! use std::env; -//! use io::IoChannel; -//! use ethcore::client::{Client, ClientConfig}; -//! use ethsync::{EthSync, SyncConfig, ManageNetwork, NetworkConfiguration}; -//! use ethcore::ethereum; -//! use ethcore::miner::{GasPricer, Miner}; -//! -//! fn main() { -//! let dir = env::temp_dir(); -//! let spec = ethereum::new_frontier(); -//! let miner = Miner::new( -//! Default::default(), -//! GasPricer::new_fixed(20_000_000_000u64.into()), -//! &spec, -//! None -//! ); -//! let client = Client::new( -//! ClientConfig::default(), -//! &spec, -//! &dir, -//! miner, -//! IoChannel::disconnected() -//! ).unwrap(); -//! let sync = EthSync::new(SyncConfig::default(), client, NetworkConfiguration::from(NetworkConfiguration::new())).unwrap(); -//! sync.start_network(); -//! } -//! ``` extern crate ethcore_network as network; extern crate ethcore_io as io; @@ -83,6 +49,7 @@ extern crate ethcore_ipc as ipc; mod chain; mod blocks; mod sync_io; +mod snapshot; #[cfg(test)] mod tests; @@ -96,4 +63,3 @@ pub use api::{EthSync, SyncProvider, SyncClient, NetworkManagerClient, ManageNet ServiceConfiguration, NetworkConfiguration}; pub use chain::{SyncStatus, SyncState}; pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError}; - diff --git a/sync/src/snapshot.rs b/sync/src/snapshot.rs new file mode 100644 index 000000000..ca9adf220 --- /dev/null +++ b/sync/src/snapshot.rs @@ -0,0 +1,200 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + + +use util::{H256, Hashable}; +use std::collections::HashSet; +use ethcore::snapshot::ManifestData; + +#[derive(PartialEq, Eq, Debug)] +pub enum ChunkType { + State(H256), + Block(H256), +} + +pub struct Snapshot { + pending_state_chunks: Vec, + pending_block_chunks: Vec, + downloading_chunks: HashSet, + completed_chunks: HashSet, + snapshot_hash: Option, +} + +impl Snapshot { + /// Create a new instance. + pub fn new() -> Snapshot { + Snapshot { + pending_state_chunks: Vec::new(), + pending_block_chunks: Vec::new(), + downloading_chunks: HashSet::new(), + completed_chunks: HashSet::new(), + snapshot_hash: None, + } + } + + /// Clear everything. + pub fn clear(&mut self) { + self.pending_state_chunks.clear(); + self.pending_block_chunks.clear(); + self.downloading_chunks.clear(); + self.completed_chunks.clear(); + self.snapshot_hash = None; + } + + /// Reset collection for a manifest RLP + pub fn reset_to(&mut self, manifest: &ManifestData, hash: &H256) { + self.clear(); + self.pending_state_chunks = manifest.state_hashes.clone(); + self.pending_block_chunks = manifest.block_hashes.clone(); + self.snapshot_hash = Some(hash.clone()); + } + + /// Validate chunk and mark it as downloaded + pub fn validate_chunk(&mut self, chunk: &[u8]) -> Result { + let hash = chunk.sha3(); + if self.completed_chunks.contains(&hash) { + trace!(target: "sync", "Ignored proccessed chunk: {}", hash.hex()); + return Err(()); + } + self.downloading_chunks.remove(&hash); + if self.pending_block_chunks.iter().any(|h| h == &hash) { + self.completed_chunks.insert(hash.clone()); + return Ok(ChunkType::Block(hash)); + } + if self.pending_state_chunks.iter().any(|h| h == &hash) { + self.completed_chunks.insert(hash.clone()); + return Ok(ChunkType::State(hash)); + } + trace!(target: "sync", "Ignored unknown chunk: {}", hash.hex()); + Err(()) + } + + /// Find a chunk to download + pub fn needed_chunk(&mut self) -> Option { + // check state chunks first + let mut chunk = self.pending_state_chunks.iter() + .find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h)) + .cloned(); + if chunk.is_none() { + chunk = self.pending_block_chunks.iter() + .find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h)) + .cloned(); + } + + if let Some(hash) = chunk { + self.downloading_chunks.insert(hash.clone()); + } + chunk + } + + pub fn clear_chunk_download(&mut self, hash: &H256) { + self.downloading_chunks.remove(hash); + } + + pub fn snapshot_hash(&self) -> Option { + self.snapshot_hash + } + + pub fn total_chunks(&self) -> usize { + self.pending_block_chunks.len() + self.pending_state_chunks.len() + } + + pub fn done_chunks(&self) -> usize { + self.total_chunks() - self.completed_chunks.len() + } + + pub fn is_complete(&self) -> bool { + self.total_chunks() == self.completed_chunks.len() + } +} + +#[cfg(test)] +mod test { + use util::*; + use super::*; + use ethcore::snapshot::ManifestData; + + fn is_empty(snapshot: &Snapshot) -> bool { + snapshot.pending_block_chunks.is_empty() && + snapshot.pending_state_chunks.is_empty() && + snapshot.completed_chunks.is_empty() && + snapshot.downloading_chunks.is_empty() && + snapshot.snapshot_hash.is_none() + } + + fn test_manifest() -> (ManifestData, H256, Vec, Vec) { + let state_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); + let block_chunks: Vec = (0..20).map(|_| H256::random().to_vec()).collect(); + let manifest = ManifestData { + state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), + block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), + state_root: H256::new(), + block_number: 42, + block_hash: H256::new(), + }; + let mhash = manifest.clone().into_rlp().sha3(); + (manifest, mhash, state_chunks, block_chunks) + } + + #[test] + fn create_clear() { + let mut snapshot = Snapshot::new(); + assert!(is_empty(&snapshot)); + let (manifest, mhash, _, _,) = test_manifest(); + snapshot.reset_to(&manifest, &mhash); + assert!(!is_empty(&snapshot)); + snapshot.clear(); + assert!(is_empty(&snapshot)); + } + + #[test] + fn validate_chunks() { + let mut snapshot = Snapshot::new(); + let (manifest, mhash, state_chunks, block_chunks) = test_manifest(); + snapshot.reset_to(&manifest, &mhash); + assert!(snapshot.validate_chunk(&H256::random().to_vec()).is_err()); + + let requested: Vec = (0..40).map(|_| snapshot.needed_chunk().unwrap()).collect(); + assert!(snapshot.needed_chunk().is_none()); + assert_eq!(&requested[0..20], &manifest.state_hashes[..]); + assert_eq!(&requested[20..40], &manifest.block_hashes[..]); + assert_eq!(snapshot.downloading_chunks.len(), 40); + + assert_eq!(snapshot.validate_chunk(&state_chunks[4]), Ok(ChunkType::State(manifest.state_hashes[4].clone()))); + assert_eq!(snapshot.completed_chunks.len(), 1); + assert_eq!(snapshot.downloading_chunks.len(), 39); + + assert_eq!(snapshot.validate_chunk(&block_chunks[10]), Ok(ChunkType::Block(manifest.block_hashes[10].clone()))); + assert_eq!(snapshot.completed_chunks.len(), 2); + assert_eq!(snapshot.downloading_chunks.len(), 38); + + for (i, data) in state_chunks.iter().enumerate() { + if i != 4 { + assert!(snapshot.validate_chunk(data).is_ok()); + } + } + + for (i, data) in block_chunks.iter().enumerate() { + if i != 10 { + assert!(snapshot.validate_chunk(data).is_ok()); + } + } + + assert!(snapshot.is_complete()); + assert_eq!(snapshot.snapshot_hash(), Some(manifest.into_rlp().sha3())); + } +} + diff --git a/sync/src/sync_io.rs b/sync/src/sync_io.rs index 91070adc5..fa95941ea 100644 --- a/sync/src/sync_io.rs +++ b/sync/src/sync_io.rs @@ -16,6 +16,8 @@ use network::{NetworkContext, PeerId, PacketId, NetworkError}; use ethcore::client::BlockChainClient; +use ethcore::snapshot::SnapshotService; +use api::ETH_PROTOCOL; /// IO interface for the syning handler. /// Provides peer connection management and an interface to the blockchain client. @@ -31,10 +33,14 @@ pub trait SyncIo { fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec) -> Result<(), NetworkError>; /// Get the blockchain fn chain(&self) -> &BlockChainClient; + /// Get the snapshot service. + fn snapshot_service(&self) -> &SnapshotService; /// Returns peer client identifier string fn peer_info(&self, peer_id: PeerId) -> String { peer_id.to_string() } + /// Maximum mutuallt supported ETH protocol version + fn eth_protocol_version(&self, peer_id: PeerId) -> u8; /// Returns if the chain block queue empty fn is_chain_queue_empty(&self) -> bool { self.chain().queue_info().is_empty() @@ -46,15 +52,17 @@ pub trait SyncIo { /// Wraps `NetworkContext` and the blockchain client pub struct NetSyncIo<'s, 'h> where 'h: 's { network: &'s NetworkContext<'h>, - chain: &'s BlockChainClient + chain: &'s BlockChainClient, + snapshot_service: &'s SnapshotService, } impl<'s, 'h> NetSyncIo<'s, 'h> { /// Creates a new instance from the `NetworkContext` and the blockchain client reference. - pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient) -> NetSyncIo<'s, 'h> { + pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient, snapshot_service: &'s SnapshotService) -> NetSyncIo<'s, 'h> { NetSyncIo { network: network, chain: chain, + snapshot_service: snapshot_service, } } } @@ -80,6 +88,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { self.chain } + fn snapshot_service(&self) -> &SnapshotService { + self.snapshot_service + } + fn peer_info(&self, peer_id: PeerId) -> String { self.network.peer_info(peer_id) } @@ -87,6 +99,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { fn is_expired(&self) -> bool { self.network.is_expired() } + + fn eth_protocol_version(&self, peer_id: PeerId) -> u8 { + self.network.protocol_version(peer_id, ETH_PROTOCOL).unwrap_or(0) + } } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index fba57681d..cbed49eff 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -16,22 +16,26 @@ use util::*; use network::*; +use tests::snapshot::*; use ethcore::client::{TestBlockChainClient, BlockChainClient}; use ethcore::header::BlockNumber; +use ethcore::snapshot::SnapshotService; use sync_io::SyncIo; use chain::ChainSync; use ::SyncConfig; pub struct TestIo<'p> { pub chain: &'p mut TestBlockChainClient, + pub snapshot_service: &'p TestSnapshotService, pub queue: &'p mut VecDeque, pub sender: Option, } impl<'p> TestIo<'p> { - pub fn new(chain: &'p mut TestBlockChainClient, queue: &'p mut VecDeque, sender: Option) -> TestIo<'p> { + pub fn new(chain: &'p mut TestBlockChainClient, ss: &'p TestSnapshotService, queue: &'p mut VecDeque, sender: Option) -> TestIo<'p> { TestIo { chain: chain, + snapshot_service: ss, queue: queue, sender: sender } @@ -70,6 +74,14 @@ impl<'p> SyncIo for TestIo<'p> { fn chain(&self) -> &BlockChainClient { self.chain } + + fn snapshot_service(&self) -> &SnapshotService { + self.snapshot_service + } + + fn eth_protocol_version(&self, _peer: PeerId) -> u8 { + 64 + } } pub struct TestPacket { @@ -80,6 +92,7 @@ pub struct TestPacket { pub struct TestPeer { pub chain: TestBlockChainClient, + pub snapshot_service: Arc, pub sync: RwLock, pub queue: VecDeque, } @@ -103,9 +116,11 @@ impl TestNet { let chain = TestBlockChainClient::new(); let mut config = SyncConfig::default(); config.fork_block = fork; + let ss = Arc::new(TestSnapshotService::new()); let sync = ChainSync::new(config, &chain); net.peers.push(TestPeer { sync: RwLock::new(sync), + snapshot_service: ss, chain: chain, queue: VecDeque::new(), }); @@ -126,7 +141,7 @@ impl TestNet { for client in 0..self.peers.len() { if peer != client { let mut p = self.peers.get_mut(peer).unwrap(); - p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(client as PeerId)), client as PeerId); + p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(client as PeerId)), client as PeerId); } } } @@ -137,22 +152,22 @@ impl TestNet { if let Some(packet) = self.peers[peer].queue.pop_front() { let mut p = self.peers.get_mut(packet.recipient).unwrap(); trace!("--- {} -> {} ---", peer, packet.recipient); - ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data); + ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data); trace!("----------------"); } let mut p = self.peers.get_mut(peer).unwrap(); - p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &mut p.queue, None)); + p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, None)); } } pub fn sync_step_peer(&mut self, peer_num: usize) { let mut peer = self.peer_mut(peer_num); - peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None)); + peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None)); } pub fn restart_peer(&mut self, i: usize) { let peer = self.peer_mut(i); - peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None)); + peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None)); } pub fn sync(&mut self) -> u32 { @@ -181,6 +196,6 @@ impl TestNet { pub fn trigger_chain_new_blocks(&mut self, peer_id: usize) { let mut peer = self.peer_mut(peer_id); - peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None), &[], &[], &[], &[], &[]); + peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None), &[], &[], &[], &[], &[]); } } diff --git a/sync/src/tests/mod.rs b/sync/src/tests/mod.rs index 5afda05f0..bdb4ae4f9 100644 --- a/sync/src/tests/mod.rs +++ b/sync/src/tests/mod.rs @@ -15,5 +15,6 @@ // along with Parity. If not, see . pub mod helpers; +pub mod snapshot; mod chain; mod rpc; diff --git a/sync/src/tests/snapshot.rs b/sync/src/tests/snapshot.rs new file mode 100644 index 000000000..b27602b0d --- /dev/null +++ b/sync/src/tests/snapshot.rs @@ -0,0 +1,123 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use util::*; +use ethcore::snapshot::{SnapshotService, ManifestData, RestorationStatus}; +use ethcore::header::BlockNumber; +use ethcore::client::{EachBlockWith}; +use super::helpers::*; + +pub struct TestSnapshotService { + manifest: Option, + chunks: HashMap, + + restoration_manifest: Mutex>, + state_restoration_chunks: Mutex>, + block_restoration_chunks: Mutex>, +} + +impl TestSnapshotService { + pub fn new() -> TestSnapshotService { + TestSnapshotService { + manifest: None, + chunks: HashMap::new(), + restoration_manifest: Mutex::new(None), + state_restoration_chunks: Mutex::new(HashMap::new()), + block_restoration_chunks: Mutex::new(HashMap::new()), + } + } + + pub fn new_with_snapshot(num_chunks: usize, block_hash: H256, block_number: BlockNumber) -> TestSnapshotService { + let num_state_chunks = num_chunks / 2; + let num_block_chunks = num_chunks - num_state_chunks; + let state_chunks: Vec = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect(); + let block_chunks: Vec = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect(); + let manifest = ManifestData { + state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(), + block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(), + state_root: H256::new(), + block_number: block_number, + block_hash: block_hash, + }; + let mut chunks: HashMap = state_chunks.into_iter().map(|data| (data.sha3(), data)).collect(); + chunks.extend(block_chunks.into_iter().map(|data| (data.sha3(), data))); + TestSnapshotService { + manifest: Some(manifest), + chunks: chunks, + restoration_manifest: Mutex::new(None), + state_restoration_chunks: Mutex::new(HashMap::new()), + block_restoration_chunks: Mutex::new(HashMap::new()), + } + } +} + +impl SnapshotService for TestSnapshotService { + fn manifest(&self) -> Option { + self.manifest.as_ref().cloned() + } + + fn chunk(&self, hash: H256) -> Option { + self.chunks.get(&hash).cloned() + } + + fn status(&self) -> RestorationStatus { + match &*self.restoration_manifest.lock() { + &Some(ref manifest) if self.state_restoration_chunks.lock().len() == manifest.state_hashes.len() && + self.block_restoration_chunks.lock().len() == manifest.block_hashes.len() => RestorationStatus::Inactive, + &Some(_) => RestorationStatus::Ongoing { + state_chunks_done: self.state_restoration_chunks.lock().len() as u32, + block_chunks_done: self.block_restoration_chunks.lock().len() as u32, + }, + &None => RestorationStatus::Inactive, + } + } + + fn begin_restore(&self, manifest: ManifestData) { + *self.restoration_manifest.lock() = Some(manifest); + self.state_restoration_chunks.lock().clear(); + self.block_restoration_chunks.lock().clear(); + } + + fn abort_restore(&self) { + *self.restoration_manifest.lock() = None; + self.state_restoration_chunks.lock().clear(); + self.block_restoration_chunks.lock().clear(); + } + + fn restore_state_chunk(&self, hash: H256, chunk: Bytes) { + if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.state_hashes.iter().any(|h| h == &hash)) { + self.state_restoration_chunks.lock().insert(hash, chunk); + } + } + + fn restore_block_chunk(&self, hash: H256, chunk: Bytes) { + if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.block_hashes.iter().any(|h| h == &hash)) { + self.block_restoration_chunks.lock().insert(hash, chunk); + } + } +} + +#[test] +fn snapshot_sync() { + ::env_logger::init().ok(); + let mut net = TestNet::new(2); + net.peer_mut(0).snapshot_service = Arc::new(TestSnapshotService::new_with_snapshot(16, H256::new(), 1)); + net.peer_mut(0).chain.add_blocks(1, EachBlockWith::Nothing); + net.sync_steps(19); // status + manifest + chunks + assert_eq!(net.peer(1).snapshot_service.state_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().state_hashes.len()); + assert_eq!(net.peer(1).snapshot_service.block_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().block_hashes.len()); +} + diff --git a/util/network/src/host.rs b/util/network/src/host.rs index 359f54f1a..ebc10324f 100644 --- a/util/network/src/host.rs +++ b/util/network/src/host.rs @@ -282,6 +282,12 @@ impl<'s> NetworkContext<'s> { } "unknown".to_owned() } + + /// Returns max version for a given protocol. + pub fn protocol_version(&self, peer: PeerId, protocol: &str) -> Option { + let session = self.resolve_session(peer); + session.and_then(|s| s.lock().capability_version(protocol)) + } } /// Shared host information diff --git a/util/network/src/session.rs b/util/network/src/session.rs index 8ebd37090..164248d62 100644 --- a/util/network/src/session.rs +++ b/util/network/src/session.rs @@ -243,6 +243,11 @@ impl Session { self.info.capabilities.iter().any(|c| c.protocol == protocol) } + /// Checks if peer supports given capability + pub fn capability_version(&self, protocol: &str) -> Option { + self.info.capabilities.iter().filter_map(|c| if c.protocol == protocol { Some(c.version) } else { None }).max() + } + /// Register the session socket with the event loop pub fn register_socket>(&self, reg: Token, event_loop: &mut EventLoop) -> Result<(), NetworkError> { if self.expired() { diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 5db7801a1..177df5fa0 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -16,9 +16,11 @@ //! Key-Value store abstraction with `RocksDB` backend. +use std::io::ErrorKind; use common::*; use elastic_array::*; use std::default::Default; +use std::path::PathBuf; use rlp::{UntrustedRlp, RlpType, View, Compressible}; use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator, Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column}; @@ -189,12 +191,18 @@ impl<'a> Iterator for DatabaseIterator { } } +struct DBAndColumns { + db: DB, + cfs: Vec, +} + /// Key-Value database. pub struct Database { - db: DB, + db: RwLock>, + config: DatabaseConfig, write_opts: WriteOptions, - cfs: Vec, overlay: RwLock, KeyState>>>, + path: String, } impl Database { @@ -278,11 +286,13 @@ impl Database { }, Err(s) => { return Err(s); } }; + let num_cols = cfs.len(); Ok(Database { - db: db, + db: RwLock::new(Some(DBAndColumns{ db: db, cfs: cfs })), + config: config.clone(), write_opts: write_opts, - overlay: RwLock::new((0..(cfs.len() + 1)).map(|_| HashMap::new()).collect()), - cfs: cfs, + overlay: RwLock::new((0..(num_cols + 1)).map(|_| HashMap::new()).collect()), + path: path.to_owned(), }) } @@ -320,94 +330,167 @@ impl Database { /// Commit buffered changes to database. pub fn flush(&self) -> Result<(), String> { - let batch = WriteBatch::new(); - let mut overlay = self.overlay.write(); + match &*self.db.read() { + &Some(DBAndColumns { ref db, ref cfs }) => { + let batch = WriteBatch::new(); + let mut overlay = self.overlay.write(); - for (c, column) in overlay.iter_mut().enumerate() { - let column_data = mem::replace(column, HashMap::new()); - for (key, state) in column_data.into_iter() { - match state { - KeyState::Delete => { - if c > 0 { - try!(batch.delete_cf(self.cfs[c - 1], &key)); - } else { - try!(batch.delete(&key)); - } - }, - KeyState::Insert(value) => { - if c > 0 { - try!(batch.put_cf(self.cfs[c - 1], &key, &value)); - } else { - try!(batch.put(&key, &value)); - } - }, - KeyState::InsertCompressed(value) => { - let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks); - if c > 0 { - try!(batch.put_cf(self.cfs[c - 1], &key, &compressed)); - } else { - try!(batch.put(&key, &value)); + for (c, column) in overlay.iter_mut().enumerate() { + let column_data = mem::replace(column, HashMap::new()); + for (key, state) in column_data.into_iter() { + match state { + KeyState::Delete => { + if c > 0 { + try!(batch.delete_cf(cfs[c - 1], &key)); + } else { + try!(batch.delete(&key)); + } + }, + KeyState::Insert(value) => { + if c > 0 { + try!(batch.put_cf(cfs[c - 1], &key, &value)); + } else { + try!(batch.put(&key, &value)); + } + }, + KeyState::InsertCompressed(value) => { + let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks); + if c > 0 { + try!(batch.put_cf(cfs[c - 1], &key, &compressed)); + } else { + try!(batch.put(&key, &value)); + } + } } } } - } + db.write_opt(batch, &self.write_opts) + }, + &None => Err("Database is closed".to_owned()) } - self.db.write_opt(batch, &self.write_opts) } /// Commit transaction to database. pub fn write(&self, tr: DBTransaction) -> Result<(), String> { - let batch = WriteBatch::new(); - let ops = tr.ops; - for op in ops { - match op { - DBOp::Insert { col, key, value } => { - try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(self.cfs[c as usize], &key, &value))) - }, - DBOp::InsertCompressed { col, key, value } => { - let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks); - try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(self.cfs[c as usize], &key, &compressed))) - }, - DBOp::Delete { col, key } => { - try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(self.cfs[c as usize], &key))) - }, - } + match &*self.db.read() { + &Some(DBAndColumns { ref db, ref cfs }) => { + let batch = WriteBatch::new(); + let ops = tr.ops; + for op in ops { + match op { + DBOp::Insert { col, key, value } => { + try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(cfs[c as usize], &key, &value))) + }, + DBOp::InsertCompressed { col, key, value } => { + let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks); + try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(cfs[c as usize], &key, &compressed))) + }, + DBOp::Delete { col, key } => { + try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(cfs[c as usize], &key))) + }, + } + } + db.write_opt(batch, &self.write_opts) + }, + &None => Err("Database is closed".to_owned()) } - self.db.write_opt(batch, &self.write_opts) } /// Get value by key. pub fn get(&self, col: Option, key: &[u8]) -> Result, String> { - let overlay = &self.overlay.read()[Self::to_overlay_column(col)]; - match overlay.get(key) { - Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())), - Some(&KeyState::Delete) => Ok(None), - None => { - col.map_or_else( - || self.db.get(key).map(|r| r.map(|v| v.to_vec())), - |c| self.db.get_cf(self.cfs[c as usize], key).map(|r| r.map(|v| v.to_vec()))) + match &*self.db.read() { + &Some(DBAndColumns { ref db, ref cfs }) => { + let overlay = &self.overlay.read()[Self::to_overlay_column(col)]; + match overlay.get(key) { + Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())), + Some(&KeyState::Delete) => Ok(None), + None => { + col.map_or_else( + || db.get(key).map(|r| r.map(|v| v.to_vec())), + |c| db.get_cf(cfs[c as usize], key).map(|r| r.map(|v| v.to_vec()))) + }, + } }, + &None => Ok(None), } } /// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values. - // TODO: support prefix seek for unflushed ata + // TODO: support prefix seek for unflushed data pub fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option> { - let mut iter = col.map_or_else(|| self.db.iterator(IteratorMode::From(prefix, Direction::Forward)), - |c| self.db.iterator_cf(self.cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap()); - match iter.next() { - // TODO: use prefix_same_as_start read option (not availabele in C API currently) - Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, - _ => None + match &*self.db.read() { + &Some(DBAndColumns { ref db, ref cfs }) => { + let mut iter = col.map_or_else(|| db.iterator(IteratorMode::From(prefix, Direction::Forward)), + |c| db.iterator_cf(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap()); + match iter.next() { + // TODO: use prefix_same_as_start read option (not availabele in C API currently) + Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None }, + _ => None + } + }, + &None => None, } } /// Get database iterator for flushed data. pub fn iter(&self, col: Option) -> DatabaseIterator { //TODO: iterate over overlay - col.map_or_else(|| DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) }, - |c| DatabaseIterator { iter: self.db.iterator_cf(self.cfs[c as usize], IteratorMode::Start).unwrap() }) + match &*self.db.read() { + &Some(DBAndColumns { ref db, ref cfs }) => { + col.map_or_else(|| DatabaseIterator { iter: db.iterator(IteratorMode::Start) }, + |c| DatabaseIterator { iter: db.iterator_cf(cfs[c as usize], IteratorMode::Start).unwrap() }) + }, + &None => panic!("Not supported yet") //TODO: return an empty iterator or change return type + } + } + + /// Close the database + fn close(&self) { + *self.db.write() = None; + self.overlay.write().clear(); + } + + /// Restore the database from a copy at given path. + pub fn restore(&self, new_db: &str) -> Result<(), UtilError> { + self.close(); + + let mut backup_db = PathBuf::from(&self.path); + backup_db.pop(); + backup_db.push("backup_db"); + println!("Path at {:?}", self.path); + println!("Backup at {:?}", backup_db); + + let existed = match fs::rename(&self.path, &backup_db) { + Ok(_) => true, + Err(e) => if let ErrorKind::NotFound = e.kind() { + false + } else { + return Err(e.into()); + } + }; + + match fs::rename(&new_db, &self.path) { + Ok(_) => { + // clean up the backup. + if existed { + try!(fs::remove_dir_all(&backup_db)); + } + } + Err(e) => { + // restore the backup. + if existed { + try!(fs::rename(&backup_db, &self.path)); + } + return Err(e.into()) + } + } + + // reopen the database and steal handles into self + let db = try!(Self::open(&self.config, &self.path)); + *self.db.write() = mem::replace(&mut *db.db.write(), None); + *self.overlay.write() = mem::replace(&mut *db.overlay.write(), Vec::new()); + Ok(()) } }