grassrootseconomics/openethereum
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of github.com:ethcore/parity into spec_load_errors

Browse Source
This commit is contained in:
debris 2016-09-07 10:24:58 +02:00
commit ba806a6359
65 changed files with 2129 additions and 596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@ -310,6 +310,7 @@ version = "1.4.0"
dependencies = [
"clippy 0.0.85 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"ethcore-devtools 1.4.0",
"ethcore-rpc 1.4.0",
"ethcore-util 1.4.0",
"https-fetch 0.1.0",
@ -478,6 +479,7 @@ version = "1.4.0"
dependencies = [
"clippy 0.0.85 (registry+https://github.com/rust-lang/crates.io-index)",
"env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"ethcore-devtools 1.4.0",
"ethcore-io 1.4.0",
"ethcore-rpc 1.4.0",
"ethcore-util 1.4.0",
@ -1109,7 +1111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "parity-dapps"
version = "1.4.0"
source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a"
source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc"
dependencies = [
"aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
"glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1123,7 +1125,7 @@ dependencies = [
[[package]]
name = "parity-dapps-home"
version = "1.4.0"
source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a"
source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc"
dependencies = [
"parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
]
@ -1131,7 +1133,7 @@ dependencies = [
[[package]]
name = "parity-dapps-signer"
version = "1.4.0"
source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a"
source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc"
dependencies = [
"parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
]
@ -1139,7 +1141,7 @@ dependencies = [
[[package]]
name = "parity-dapps-status"
version = "1.4.0"
source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a"
source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc"
dependencies = [
"parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
]
@ -1147,7 +1149,7 @@ dependencies = [
[[package]]
name = "parity-dapps-wallet"
version = "1.4.0"
source = "git+https://github.com/ethcore/parity-ui.git#e4dddf36e7c9fa5c6e746790119c71f67438784a"
source = "git+https://github.com/ethcore/parity-ui.git#926b09b66c4940b09dc82c52adb4afd9e31155bc"
dependencies = [
"parity-dapps 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
]

1
dapps/Cargo.toml
View File

@ -23,6 +23,7 @@ serde_macros = { version = "0.8", optional = true }
zip = { version = "0.1", default-features = false }
ethabi = "0.2.2"
linked-hash-map = "0.3"
ethcore-devtools = { path = "../devtools" }
ethcore-rpc = { path = "../rpc" }
ethcore-util = { path = "../util" }
https-fetch = { path = "../util/https-fetch" }

87
dapps/src/apps/fetcher.rs
View File

@ -30,6 +30,7 @@ use hyper;
use hyper::status::StatusCode;
use random_filename;
use SyncStatus;
use util::{Mutex, H256};
use util::sha3::sha3;
use page::LocalPageEndpoint;
@ -44,6 +45,7 @@ const MAX_CACHED_DAPPS: usize = 10;
pub struct AppFetcher<R: URLHint = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
sync: Arc<SyncStatus>,
dapps: Arc<Mutex<ContentCache>>,
}
@ -56,13 +58,14 @@ impl<R: URLHint> Drop for AppFetcher<R> {
impl<R: URLHint> AppFetcher<R> {
pub fn new(resolver: R) -> Self {
pub fn new(resolver: R, sync_status: Arc<SyncStatus>) -> Self {
let mut dapps_path = env::temp_dir();
dapps_path.push(random_filename());
AppFetcher {
dapps_path: dapps_path,
resolver: resolver,
sync: sync_status,
dapps: Arc::new(Mutex::new(ContentCache::default())),
}
}
@ -74,14 +77,20 @@ impl<R: URLHint> AppFetcher<R> {
pub fn contains(&self, app_id: &str) -> bool {
let mut dapps = self.dapps.lock();
match dapps.get(app_id) {
// Check if we already have the app
Some(_) => true,
// fallback to resolver
None => match app_id.from_hex() {
Ok(app_id) => self.resolver.resolve(app_id).is_some(),
_ => false,
},
// Check if we already have the app
if dapps.get(app_id).is_some() {
return true;
}
// fallback to resolver
if let Ok(app_id) = app_id.from_hex() {
// if app_id is valid, but we are syncing always return true.
if self.sync.is_major_syncing() {
return true;
}
// else try to resolve the app_id
self.resolver.resolve(app_id).is_some()
} else {
false
}
}
@ -89,6 +98,15 @@ impl<R: URLHint> AppFetcher<R> {
let mut dapps = self.dapps.lock();
let app_id = path.app_id.clone();
if self.sync.is_major_syncing() {
return Box::new(ContentHandler::error(
StatusCode::ServiceUnavailable,
"Sync In Progress",
"Your node is still syncing. We cannot resolve any content before it's fully synced.",
Some("<a href=\"javascript:window.location.reload()\">Refresh</a>")
));
}
let (new_status, handler) = {
let status = dapps.get(&app_id);
match status {
@ -98,32 +116,42 @@ impl<R: URLHint> AppFetcher<R> {
},
// App is already being fetched
Some(&mut ContentStatus::Fetching(_)) => {
(None, Box::new(ContentHandler::html(
(None, Box::new(ContentHandler::error_with_refresh(
StatusCode::ServiceUnavailable,
format!(
"<html><head>{}</head><body>{}</body></html>",
"<meta http-equiv=\"refresh\" content=\"1\">",
"<h1>This dapp is already being downloaded.</h1><h2>Please wait...</h2>",
)
"Download In Progress",
"This dapp is already being downloaded. Please wait...",
None,
)) as Box<Handler>)
},
// We need to start fetching app
None => {
let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let app = self.resolver.resolve(app_hex).expect("to_handler is called only when `contains` returns true.");
let abort = Arc::new(AtomicBool::new(false));
let app = self.resolver.resolve(app_hex);
(Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new(
app,
abort,
control,
path.using_dapps_domains,
DappInstaller {
dapp_id: app_id.clone(),
dapps_path: self.dapps_path.clone(),
dapps: self.dapps.clone(),
}
)) as Box<Handler>)
if let Some(app) = app {
let abort = Arc::new(AtomicBool::new(false));
(Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new(
app,
abort,
control,
path.using_dapps_domains,
DappInstaller {
dapp_id: app_id.clone(),
dapps_path: self.dapps_path.clone(),
dapps: self.dapps.clone(),
}
)) as Box<Handler>)
} else {
// This may happen when sync status changes in between
// `contains` and `to_handler`
(None, Box::new(ContentHandler::error(
StatusCode::NotFound,
"Resource Not Found",
"Requested resource was not found.",
None
)) as Box<Handler>)
}
},
}
};
@ -294,6 +322,7 @@ impl ContentValidator for DappInstaller {
#[cfg(test)]
mod tests {
use std::env;
use std::sync::Arc;
use util::Bytes;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
@ -312,7 +341,7 @@ mod tests {
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = AppFetcher::new(FakeResolver);
let fetcher = AppFetcher::new(FakeResolver, Arc::new(|| false));
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),

22
dapps/src/error_tpl.html Normal file
View File

@ -0,0 +1,22 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width">
{meta}
<title>{title}</title>
<link rel="stylesheet" href="/parity-utils/styles.css">
</head>
<body>
<div class="parity-navbar">
</div>
<div class="parity-box">
<h1>{title}</h1>
<h3>{message}</h3>
<p><code>{details}</code></p>
</div>
<div class="parity-status">
<small>{version}</small>
</div>
</body>
</html>

33
dapps/src/handlers/content.rs
View File

@ -21,6 +21,8 @@ use hyper::{header, server, Decoder, Encoder, Next};
use hyper::net::HttpStream;
use hyper::status::StatusCode;
use util::version;
pub struct ContentHandler {
code: StatusCode,
content: String,
@ -38,15 +40,6 @@ impl ContentHandler {
}
}
pub fn forbidden(content: String, mimetype: String) -> Self {
ContentHandler {
code: StatusCode::Forbidden,
content: content,
mimetype: mimetype,
write_pos: 0
}
}
pub fn not_found(content: String, mimetype: String) -> Self {
ContentHandler {
code: StatusCode::NotFound,
@ -60,6 +53,28 @@ impl ContentHandler {
Self::new(code, content, "text/html".into())
}
pub fn error(code: StatusCode, title: &str, message: &str, details: Option<&str>) -> Self {
Self::html(code, format!(
include_str!("../error_tpl.html"),
title=title,
meta="",
message=message,
details=details.unwrap_or_else(|| ""),
version=version(),
))
}
pub fn error_with_refresh(code: StatusCode, title: &str, message: &str, details: Option<&str>) -> Self {
Self::html(code, format!(
include_str!("../error_tpl.html"),
title=title,
meta="<meta http-equiv=\"refresh\" content=\"1\">",
message=message,
details=details.unwrap_or_else(|| ""),
version=version(),
))
}
pub fn new(code: StatusCode, content: String, mimetype: String) -> Self {
ContentHandler {
code: code,

32
dapps/src/handlers/fetch.rs
View File

@ -121,16 +121,20 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
deadline: Instant::now() + Duration::from_secs(FETCH_TIMEOUT),
receiver: receiver,
},
Err(e) => FetchState::Error(ContentHandler::html(
Err(e) => FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
format!("<h1>Error starting dapp download.</h1><pre>{}</pre>", e),
"Unable To Start Dapp Download",
"Could not initialize download of the dapp. It might be a problem with the remote server.",
Some(&format!("{}", e)),
)),
}
},
// or return error
_ => FetchState::Error(ContentHandler::html(
_ => FetchState::Error(ContentHandler::error(
StatusCode::MethodNotAllowed,
"<h1>Only <code>GET</code> requests are allowed.</h1>".into(),
"Method Not Allowed",
"Only <code>GET</code> requests are allowed.",
None,
)),
})
} else { None };
@ -147,10 +151,12 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
// Request may time out
FetchState::InProgress { ref deadline, .. } if *deadline < Instant::now() => {
trace!(target: "dapps", "Fetching dapp failed because of timeout.");
let timeout = ContentHandler::html(
let timeout = ContentHandler::error(
StatusCode::GatewayTimeout,
format!("<h1>Could not fetch app bundle within {} seconds.</h1>", FETCH_TIMEOUT),
);
"Download Timeout",
&format!("Could not fetch dapp bundle within {} seconds.", FETCH_TIMEOUT),
None
);
Self::close_client(&mut self.client);
(Some(FetchState::Error(timeout)), Next::write())
},
@ -166,9 +172,11 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
let state = match self.dapp.validate_and_install(path.clone()) {
Err(e) => {
trace!(target: "dapps", "Error while validating dapp: {:?}", e);
FetchState::Error(ContentHandler::html(
FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
format!("<h1>Downloaded bundle does not contain valid app.</h1><pre>{}</pre>", e),
"Invalid Dapp",
"Downloaded bundle does not contain a valid dapp.",
Some(&format!("{:?}", e))
))
},
Ok(manifest) => FetchState::Done(manifest)
@ -180,9 +188,11 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
},
Ok(Err(e)) => {
warn!(target: "dapps", "Unable to fetch new dapp: {:?}", e);
let error = ContentHandler::html(
let error = ContentHandler::error(
StatusCode::BadGateway,
"<h1>There was an error when fetching the dapp.</h1>".into(),
"Download Error",
"There was an error when fetching the dapp.",
Some(&format!("{:?}", e)),
);
(Some(FetchState::Error(error)), Next::write())
},

28
dapps/src/lib.rs
View File

@ -62,6 +62,8 @@ extern crate https_fetch;
extern crate ethcore_rpc;
extern crate ethcore_util as util;
extern crate linked_hash_map;
#[cfg(test)]
extern crate ethcore_devtools as devtools;
mod endpoint;
mod apps;
@ -87,11 +89,22 @@ use ethcore_rpc::Extendable;
static DAPPS_DOMAIN : &'static str = ".parity";
/// Indicates sync status
pub trait SyncStatus: Send + Sync {
/// Returns true if there is a major sync happening.
fn is_major_syncing(&self) -> bool;
}
impl<F> SyncStatus for F where F: Fn() -> bool + Send + Sync {
fn is_major_syncing(&self) -> bool { self() }
}
/// Webapps HTTP+RPC server build.
pub struct ServerBuilder {
dapps_path: String,
handler: Arc<IoHandler>,
registrar: Arc<ContractClient>,
sync_status: Arc<SyncStatus>,
}
impl Extendable for ServerBuilder {
@ -107,9 +120,15 @@ impl ServerBuilder {
dapps_path: dapps_path,
handler: Arc::new(IoHandler::new()),
registrar: registrar,
sync_status: Arc::new(|| false),
}
}
/// Change default sync status.
pub fn with_sync_status(&mut self, status: Arc<SyncStatus>) {
self.sync_status = status;
}
/// Asynchronously start server with no authentication,
/// returns result with `Server` handle on success or an error.
pub fn start_unsecured_http(&self, addr: &SocketAddr, hosts: Option<Vec<String>>) -> Result<Server, ServerError> {
@ -119,7 +138,8 @@ impl ServerBuilder {
NoAuth,
self.handler.clone(),
self.dapps_path.clone(),
self.registrar.clone()
self.registrar.clone(),
self.sync_status.clone(),
)
}
@ -132,7 +152,8 @@ impl ServerBuilder {
HttpBasicAuth::single_user(username, password),
self.handler.clone(),
self.dapps_path.clone(),
self.registrar.clone()
self.registrar.clone(),
self.sync_status.clone(),
)
}
}
@ -166,10 +187,11 @@ impl Server {
handler: Arc<IoHandler>,
dapps_path: String,
registrar: Arc<ContractClient>,
sync_status: Arc<SyncStatus>,
) -> Result<Server, ServerError> {
let panic_handler = Arc::new(Mutex::new(None));
let authorization = Arc::new(authorization);
let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::new(apps::urlhint::URLHintContract::new(registrar)));
let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status));
let endpoints = Arc::new(apps::all_endpoints(dapps_path));
let special = Arc::new({
let mut special = HashMap::new();

2
dapps/src/page/builtin.rs
View File

@ -79,7 +79,7 @@ impl<T: WebApp> Endpoint for PageEndpoint<T> {
app: BuiltinDapp::new(self.app.clone()),
prefix: self.prefix.clone(),
path: path,
file: None,
file: Default::default(),
safe_to_embed: self.safe_to_embed,
})
}

52
dapps/src/page/handler.rs
View File

@ -22,6 +22,7 @@ use hyper::net::HttpStream;
use hyper::status::StatusCode;
use hyper::{Decoder, Encoder, Next};
use endpoint::EndpointPath;
use handlers::ContentHandler;
/// Represents a file that can be sent to client.
/// Implementation should keep track of bytes already sent internally.
@ -48,6 +49,25 @@ pub trait Dapp: Send + 'static {
fn file(&self, path: &str) -> Option<Self::DappFile>;
}
/// Currently served by `PageHandler` file
pub enum ServedFile<T: Dapp> {
/// File from dapp
File(T::DappFile),
/// Error (404)
Error(ContentHandler),
}
impl<T: Dapp> Default for ServedFile<T> {
fn default() -> Self {
ServedFile::Error(ContentHandler::error(
StatusCode::NotFound,
"404 Not Found",
"Requested dapp resource was not found.",
None
))
}
}
/// A handler for a single webapp.
/// Resolves correct paths and serves as a plumbing code between
/// hyper server and dapp.
@ -55,7 +75,7 @@ pub struct PageHandler<T: Dapp> {
/// A Dapp.
pub app: T,
/// File currently being served (or `None` if file does not exist).
pub file: Option<T::DappFile>,
pub file: ServedFile<T>,
/// Optional prefix to strip from path.
pub prefix: Option<String>,
/// Requested path.
@ -95,7 +115,7 @@ impl<T: Dapp> server::Handler<HttpStream> for PageHandler<T> {
self.app.file(&self.extract_path(url.path()))
},
_ => None,
};
}.map_or_else(|| ServedFile::default(), |f| ServedFile::File(f));
Next::write()
}
@ -104,24 +124,26 @@ impl<T: Dapp> server::Handler<HttpStream> for PageHandler<T> {
}
fn on_response(&mut self, res: &mut server::Response) -> Next {
if let Some(ref f) = self.file {
res.set_status(StatusCode::Ok);
res.headers_mut().set(header::ContentType(f.content_type().parse().unwrap()));
if !self.safe_to_embed {
res.headers_mut().set_raw("X-Frame-Options", vec![b"SAMEORIGIN".to_vec()]);
match self.file {
ServedFile::File(ref f) => {
res.set_status(StatusCode::Ok);
res.headers_mut().set(header::ContentType(f.content_type().parse().unwrap()));
if !self.safe_to_embed {
res.headers_mut().set_raw("X-Frame-Options", vec![b"SAMEORIGIN".to_vec()]);
}
Next::write()
},
ServedFile::Error(ref mut handler) => {
handler.on_response(res)
}
Next::write()
} else {
res.set_status(StatusCode::NotFound);
Next::write()
}
}
fn on_response_writable(&mut self, encoder: &mut Encoder<HttpStream>) -> Next {
match self.file {
None => Next::end(),
Some(ref f) if f.is_drained() => Next::end(),
Some(ref mut f) => match encoder.write(f.next_chunk()) {
ServedFile::Error(ref mut handler) => handler.on_response_writable(encoder),
ServedFile::File(ref f) if f.is_drained() => Next::end(),
ServedFile::File(ref mut f) => match encoder.write(f.next_chunk()) {
Ok(bytes) => {
f.bytes_written(bytes);
Next::write()
@ -190,7 +212,7 @@ fn should_extract_path_with_appid() {
port: 8080,
using_dapps_domains: true,
},
file: None,
file: Default::default(),
safe_to_embed: true,
};

2
dapps/src/page/local.rs
View File

@ -49,7 +49,7 @@ impl Endpoint for LocalPageEndpoint {
app: LocalDapp::new(self.path.clone()),
prefix: None,
path: path,
file: None,
file: Default::default(),
safe_to_embed: false,
})
}

7
dapps/src/router/auth.rs
View File

@ -55,10 +55,11 @@ impl Authorization for HttpBasicAuth {
match auth {
Access::Denied => {
Authorized::No(Box::new(ContentHandler::new(
Authorized::No(Box::new(ContentHandler::error(
status::StatusCode::Unauthorized,
"<h1>Unauthorized</h1>".into(),
"text/html".into(),
"Unauthorized",
"You need to provide valid credentials to access this page.",
None
)))
},
Access::AuthRequired => {

12
dapps/src/router/host_validation.rs
View File

@ -16,7 +16,7 @@
use DAPPS_DOMAIN;
use hyper::{server, header};
use hyper::{server, header, StatusCode};
use hyper::net::HttpStream;
use jsonrpc_http_server::{is_host_header_valid};
@ -38,11 +38,9 @@ pub fn is_valid(request: &server::Request<HttpStream>, allowed_hosts: &[String],
}
pub fn host_invalid_response() -> Box<server::Handler<HttpStream> + Send> {
Box::new(ContentHandler::forbidden(
r#"
<h1>Request with disallowed <code>Host</code> header has been blocked.</h1>
<p>Check the URL in your browser address bar.</p>
"#.into(),
"text/html".into()
Box::new(ContentHandler::error(StatusCode::Forbidden,
"Current Host Is Disallowed",
"You are trying to access your node using incorrect address.",
Some("Use allowed URL or specify different <code>hosts</code> CLI options.")
))
}

24
dapps/src/router/mod.rs
View File

@ -24,12 +24,12 @@ use DAPPS_DOMAIN;
use std::sync::Arc;
use std::collections::HashMap;
use url::{Url, Host};
use hyper::{self, server, Next, Encoder, Decoder, Control};
use hyper::{self, server, Next, Encoder, Decoder, Control, StatusCode};
use hyper::net::HttpStream;
use apps;
use apps::fetcher::AppFetcher;
use endpoint::{Endpoint, Endpoints, EndpointPath};
use handlers::{Redirection, extract_url};
use handlers::{Redirection, extract_url, ContentHandler};
use self::auth::{Authorization, Authorized};
/// Special endpoints are accessible on every domain (every dapp)
@ -55,9 +55,16 @@ pub struct Router<A: Authorization + 'static> {
impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
fn on_request(&mut self, req: server::Request<HttpStream>) -> Next {
// Choose proper handler depending on path / domain
let url = extract_url(&req);
let endpoint = extract_endpoint(&url);
let is_utils = endpoint.1 == SpecialEndpoint::Utils;
// Validate Host header
if let Some(ref hosts) = self.allowed_hosts {
if !host_validation::is_valid(&req, hosts, self.endpoints.keys().cloned().collect()) {
let is_valid = is_utils || host_validation::is_valid(&req, hosts, self.endpoints.keys().cloned().collect());
if !is_valid {
self.handler = host_validation::host_invalid_response();
return self.handler.on_request(req);
}
@ -70,11 +77,7 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
return self.handler.on_request(req);
}
// Choose proper handler depending on path / domain
let url = extract_url(&req);
let endpoint = extract_endpoint(&url);
let control = self.control.take().expect("on_request is called only once; control is always defined at start; qed");
self.handler = match endpoint {
// First check special endpoints
(ref path, ref endpoint) if self.special.contains_key(endpoint) => {
@ -91,7 +94,12 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
// Redirection to main page (maybe 404 instead?)
(Some(ref path), _) if *req.method() == hyper::method::Method::Get => {
let address = apps::redirection_address(path.using_dapps_domains, self.main_page);
Redirection::new(address.as_str())
Box::new(ContentHandler::error(
StatusCode::NotFound,
"404 Not Found",
"Requested content was not found.",
Some(&format!("Go back to the <a href=\"{}\">Home Page</a>.", address))
))
},
// Redirect any GET request to home.
_ if *req.method() == hyper::method::Method::Get => {

2
dapps/src/tests/api.rs
View File

@ -57,7 +57,7 @@ fn should_serve_apps() {
// then
assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned());
assert_eq!(response.headers.get(0).unwrap(), "Content-Type: application/json");
assert!(response.body.contains("Parity Home Screen"));
assert!(response.body.contains("Parity Home Screen"), response.body);
}
#[test]

2
dapps/src/tests/authorization.rs
View File

@ -54,7 +54,7 @@ fn should_reject_on_invalid_auth() {
// then
assert_eq!(response.status, "HTTP/1.1 401 Unauthorized".to_owned());
assert_eq!(response.body, "15\n<h1>Unauthorized</h1>\n0\n\n".to_owned());
assert!(response.body.contains("Unauthorized"), response.body);
assert_eq!(response.headers_raw.contains("WWW-Authenticate"), false);
}

38
dapps/src/tests/fetch.rs Normal file
View File

@ -0,0 +1,38 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use tests::helpers::{serve_with_registrar, request};
#[test]
fn should_resolve_dapp() {
// given
let (server, registrar) = serve_with_registrar();
// when
let response = request(server,
"\
GET / HTTP/1.1\r\n\
Host: 1472a9e190620cdf6b31f383373e45efcfe869a820c91f9ccd7eb9fb45e4985d.parity\r\n\
Connection: close\r\n\
\r\n\
"
);
// then
assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned());
assert_eq!(registrar.calls.lock().len(), 2);
}

73
dapps/src/tests/helpers.rs
View File

@ -15,16 +15,15 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::env;
use std::io::{Read, Write};
use std::str::{self, Lines};
use std::str;
use std::sync::Arc;
use std::net::TcpStream;
use rustc_serialize::hex::{ToHex, FromHex};
use ServerBuilder;
use Server;
use apps::urlhint::ContractClient;
use util::{Bytes, Address, Mutex, ToPretty};
use devtools::http_client;
const REGISTRAR: &'static str = "8e4e9b13d4b45cb0befc93c3061b1408f67316b2";
const URLHINT: &'static str = "deadbeefcafe0000000000000000000000000000";
@ -59,65 +58,37 @@ impl ContractClient for FakeRegistrar {
}
}
pub fn serve_hosts(hosts: Option<Vec<String>>) -> Server {
pub fn init_server(hosts: Option<Vec<String>>) -> (Server, Arc<FakeRegistrar>) {
let registrar = Arc::new(FakeRegistrar::new());
let mut dapps_path = env::temp_dir();
dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading");
let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar);
builder.start_unsecured_http(&"127.0.0.1:0".parse().unwrap(), hosts).unwrap()
let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar.clone());
(
builder.start_unsecured_http(&"127.0.0.1:0".parse().unwrap(), hosts).unwrap(),
registrar,
)
}
pub fn serve_with_auth(user: &str, pass: &str) -> Server {
let registrar = Arc::new(FakeRegistrar::new());
let builder = ServerBuilder::new(env::temp_dir().to_str().unwrap().into(), registrar);
let mut dapps_path = env::temp_dir();
dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading");
let builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar);
builder.start_basic_auth_http(&"127.0.0.1:0".parse().unwrap(), None, user, pass).unwrap()
}
pub fn serve_hosts(hosts: Option<Vec<String>>) -> Server {
init_server(hosts).0
}
pub fn serve_with_registrar() -> (Server, Arc<FakeRegistrar>) {
init_server(None)
}
pub fn serve() -> Server {
serve_hosts(None)
init_server(None).0
}
pub struct Response {
pub status: String,
pub headers: Vec<String>,
pub headers_raw: String,
pub body: String,
pub fn request(server: Server, request: &str) -> http_client::Response {
http_client::request(server.addr(), request)
}
pub fn read_block(lines: &mut Lines, all: bool) -> String {
let mut block = String::new();
loop {
let line = lines.next();
match line {
None => break,
Some("") if !all => break,
Some(v) => {
block.push_str(v);
block.push_str("\n");
},
}
}
block
}
pub fn request(server: Server, request: &str) -> Response {
let mut req = TcpStream::connect(server.addr()).unwrap();
req.write_all(request.as_bytes()).unwrap();
let mut response = String::new();
req.read_to_string(&mut response).unwrap();
let mut lines = response.lines();
let status = lines.next().unwrap().to_owned();
let headers_raw = read_block(&mut lines, false);
let headers = headers_raw.split('\n').map(|v| v.to_owned()).collect();
let body = read_block(&mut lines, true);
Response {
status: status,
headers: headers,
headers_raw: headers_raw,
body: body,
}
}

1
dapps/src/tests/mod.rs
View File

@ -20,6 +20,7 @@ mod helpers;
mod api;
mod authorization;
mod fetch;
mod redirection;
mod validation;

12
dapps/src/tests/redirection.rs
View File

@ -57,7 +57,7 @@ fn should_redirect_to_home_when_trailing_slash_is_missing() {
}
#[test]
fn should_redirect_to_home_on_invalid_dapp() {
fn should_display_404_on_invalid_dapp() {
// given
let server = serve();
@ -72,12 +72,12 @@ fn should_redirect_to_home_on_invalid_dapp() {
);
// then
assert_eq!(response.status, "HTTP/1.1 302 Found".to_owned());
assert_eq!(response.headers.get(0).unwrap(), "Location: /home/");
assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned());
assert!(response.body.contains("href=\"/home/"));
}
#[test]
fn should_redirect_to_home_on_invalid_dapp_with_domain() {
fn should_display_404_on_invalid_dapp_with_domain() {
// given
let server = serve();
@ -92,8 +92,8 @@ fn should_redirect_to_home_on_invalid_dapp_with_domain() {
);
// then
assert_eq!(response.status, "HTTP/1.1 302 Found".to_owned());
assert_eq!(response.headers.get(0).unwrap(), "Location: http://home.parity/");
assert_eq!(response.status, "HTTP/1.1 404 Not Found".to_owned());
assert!(response.body.contains("href=\"http://home.parity/"));
}
#[test]

23
dapps/src/tests/validation.rs
View File

@ -34,7 +34,7 @@ fn should_reject_invalid_host() {
// then
assert_eq!(response.status, "HTTP/1.1 403 Forbidden".to_owned());
assert_eq!(response.body, "85\n\n\t\t<h1>Request with disallowed <code>Host</code> header has been blocked.</h1>\n\t\t<p>Check the URL in your browser address bar.</p>\n\t\t\n0\n\n".to_owned());
assert!(response.body.contains("Current Host Is Disallowed"), response.body);
}
#[test]
@ -77,3 +77,24 @@ fn should_serve_dapps_domains() {
assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned());
}
#[test]
// NOTE [todr] This is required for error pages to be styled properly.
fn should_allow_parity_utils_even_on_invalid_domain() {
// given
let server = serve_hosts(Some(vec!["localhost:8080".into()]));
// when
let response = request(server,
"\
GET /parity-utils/styles.css HTTP/1.1\r\n\
Host: 127.0.0.1:8080\r\n\
Connection: close\r\n\
\r\n\
{}
"
);
// then
assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned());
}

64
devtools/src/http_client.rs Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::io::{Read, Write};
use std::str::{self, Lines};
use std::net::{TcpStream, SocketAddr};
pub struct Response {
pub status: String,
pub headers: Vec<String>,
pub headers_raw: String,
pub body: String,
}
pub fn read_block(lines: &mut Lines, all: bool) -> String {
let mut block = String::new();
loop {
let line = lines.next();
match line {
None => break,
Some("") if !all => break,
Some(v) => {
block.push_str(v);
block.push_str("\n");
},
}
}
block
}
pub fn request(address: &SocketAddr, request: &str) -> Response {
let mut req = TcpStream::connect(address).unwrap();
req.write_all(request.as_bytes()).unwrap();
let mut response = String::new();
req.read_to_string(&mut response).unwrap();
let mut lines = response.lines();
let status = lines.next().unwrap().to_owned();
let headers_raw = read_block(&mut lines, false);
let headers = headers_raw.split('\n').map(|v| v.to_owned()).collect();
let body = read_block(&mut lines, true);
Response {
status: status,
headers: headers,
headers_raw: headers_raw,
body: body,
}
}

1
devtools/src/lib.rs
View File

@ -22,6 +22,7 @@ extern crate rand;
mod random_path;
mod test_socket;
mod stop_guard;
pub mod http_client;
pub use random_path::*;
pub use test_socket::*;

1
ethcore/build.rs
View File

@ -19,5 +19,6 @@ extern crate ethcore_ipc_codegen;
fn main() {
ethcore_ipc_codegen::derive_binary("src/types/mod.rs.in").unwrap();
ethcore_ipc_codegen::derive_ipc("src/client/traits.rs").unwrap();
ethcore_ipc_codegen::derive_ipc("src/snapshot/snapshot_service_trait.rs").unwrap();
ethcore_ipc_codegen::derive_ipc("src/client/chain_notify.rs").unwrap();
}

2
ethcore/src/block_queue.rs
View File

@ -37,7 +37,7 @@ const MIN_MEM_LIMIT: usize = 16384;
const MIN_QUEUE_LIMIT: usize = 512;
/// Block queue configuration
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone)]
pub struct BlockQueueConfig {
/// Maximum number of blocks to keep in unverified queue.
/// When the limit is reached, is_full returns true.

2
ethcore/src/blockchain/config.rs
View File

@ -17,7 +17,7 @@
//! Blockchain configuration.
/// Blockchain configuration.
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone)]
pub struct Config {
/// Preferred cache size in bytes.
pub pref_cache_size: usize,

2
ethcore/src/client/chain_notify.rs
View File

@ -20,7 +20,7 @@ use util::H256;
/// Represents what has to be handled by actor listening to chain events
#[derive(Ipc)]
pub trait ChainNotify : Send + Sync {
/// fires when chain has new blocks
/// fires when chain has new blocks.
fn new_blocks(&self,
_imported: Vec<H256>,
_invalid: Vec<H256>,

199
ethcore/src/client/client.rs
View File

@ -32,7 +32,7 @@ use util::kvdb::*;
// other
use io::*;
use views::{BlockView, HeaderView, BodyView};
use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult};
use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError};
use header::BlockNumber;
use state::State;
use spec::Spec;
@ -122,11 +122,13 @@ impl SleepState {
/// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue.
pub struct Client {
mode: Mode,
chain: Arc<BlockChain>,
tracedb: Arc<TraceDB<BlockChain>>,
chain: RwLock<Arc<BlockChain>>,
tracedb: RwLock<TraceDB<BlockChain>>,
engine: Arc<Engine>,
db: Arc<Database>,
state_db: Mutex<Box<JournalDB>>,
config: ClientConfig,
db: RwLock<Arc<Database>>,
pruning: journaldb::Algorithm,
state_db: RwLock<Box<JournalDB>>,
block_queue: BlockQueue,
report: RwLock<ClientReport>,
import_lock: Mutex<()>,
@ -168,8 +170,8 @@ impl Client {
db_config.wal = config.db_wal;
let db = Arc::new(try!(Database::open(&db_config, &path.to_str().unwrap()).map_err(ClientError::Database)));
let chain = Arc::new(BlockChain::new(config.blockchain, &gb, db.clone()));
let tracedb = Arc::new(try!(TraceDB::new(config.tracing, db.clone(), chain.clone())));
let chain = Arc::new(BlockChain::new(config.blockchain.clone(), &gb, db.clone()));
let tracedb = RwLock::new(try!(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone())));
let mut state_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
if state_db.is_empty() && try!(spec.ensure_db_good(state_db.as_hashdb_mut())) {
@ -184,32 +186,34 @@ impl Client {
let engine = spec.engine.clone();
let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel.clone());
let block_queue = BlockQueue::new(config.queue.clone(), engine.clone(), message_channel.clone());
let panic_handler = PanicHandler::new_in_arc();
panic_handler.forward_from(&block_queue);
let awake = match config.mode { Mode::Dark(..) => false, _ => true };
let factories = Factories {
vm: EvmFactory::new(config.vm_type),
trie: TrieFactory::new(config.trie_spec),
vm: EvmFactory::new(config.vm_type.clone()),
trie: TrieFactory::new(config.trie_spec.clone()),
accountdb: Default::default(),
};
let client = Client {
sleep_state: Mutex::new(SleepState::new(awake)),
liveness: AtomicBool::new(awake),
mode: config.mode,
chain: chain,
mode: config.mode.clone(),
chain: RwLock::new(chain),
tracedb: tracedb,
engine: engine,
db: db,
state_db: Mutex::new(state_db),
pruning: config.pruning.clone(),
verifier: verification::new(config.verifier_type.clone()),
config: config,
db: RwLock::new(db),
state_db: RwLock::new(state_db),
block_queue: block_queue,
report: RwLock::new(Default::default()),
import_lock: Mutex::new(()),
panic_handler: panic_handler,
verifier: verification::new(config.verifier_type),
miner: miner,
io_channel: message_channel,
notify: RwLock::new(Vec::new()),
@ -253,8 +257,9 @@ impl Client {
let mut last_hashes = LastHashes::new();
last_hashes.resize(256, H256::default());
last_hashes[0] = parent_hash;
let chain = self.chain.read();
for i in 0..255 {
match self.chain.block_details(&last_hashes[i]) {
match chain.block_details(&last_hashes[i]) {
Some(details) => {
last_hashes[i + 1] = details.parent.clone();
},
@ -270,22 +275,23 @@ impl Client {
let engine = &*self.engine;
let header = &block.header;
let chain = self.chain.read();
// Check the block isn't so old we won't be able to enact it.
let best_block_number = self.chain.best_block_number();
let best_block_number = chain.best_block_number();
if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY {
warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number);
return Err(());
}
// Verify Block Family
let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &*self.chain);
let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &**chain);
if let Err(e) = verify_family_result {
warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
return Err(());
};
// Check if Parent is in chain
let chain_has_parent = self.chain.block_header(header.parent_hash());
let chain_has_parent = chain.block_header(header.parent_hash());
if let None = chain_has_parent {
warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash());
return Err(());
@ -294,9 +300,9 @@ impl Client {
// Enact Verified Block
let parent = chain_has_parent.unwrap();
let last_hashes = self.build_last_hashes(header.parent_hash().clone());
let db = self.state_db.lock().boxed_clone();
let db = self.state_db.read().boxed_clone();
let enact_result = enact_verified(block, engine, self.tracedb.tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
if let Err(e) = enact_result {
warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
return Err(());
@ -408,17 +414,18 @@ impl Client {
}
}
self.db.flush().expect("DB flush failed.");
self.db.read().flush().expect("DB flush failed.");
imported
}
fn commit_block<B>(&self, block: B, hash: &H256, block_data: &[u8]) -> ImportRoute where B: IsBlock + Drain {
let number = block.header().number();
let parent = block.header().parent_hash().clone();
let chain = self.chain.read();
// Are we committing an era?
let ancient = if number >= HISTORY {
let n = number - HISTORY;
Some((n, self.chain.block_hash(n).unwrap()))
Some((n, chain.block_hash(n).unwrap()))
} else {
None
};
@ -432,14 +439,14 @@ impl Client {
//let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new));
let mut batch = DBTransaction::new(&self.db);
let mut batch = DBTransaction::new(&self.db.read());
// CHECK! I *think* this is fine, even if the state_root is equal to another
// already-imported block of the same number.
// TODO: Prove it with a test.
block.drain().commit(&mut batch, number, hash, ancient).expect("DB commit failed.");
let route = self.chain.insert_block(&mut batch, block_data, receipts);
self.tracedb.import(&mut batch, TraceImportRequest {
let route = chain.insert_block(&mut batch, block_data, receipts);
self.tracedb.read().import(&mut batch, TraceImportRequest {
traces: traces.into(),
block_hash: hash.clone(),
block_number: number,
@ -447,8 +454,8 @@ impl Client {
retracted: route.retracted.len()
});
// Final commit to the DB
self.db.write_buffered(batch);
self.chain.commit();
self.db.read().write_buffered(batch);
chain.commit();
self.update_last_hashes(&parent, hash);
route
@ -491,10 +498,10 @@ impl Client {
};
self.block_header(id).and_then(|header| {
let db = self.state_db.lock().boxed_clone();
let db = self.state_db.read().boxed_clone();
// early exit for pruned blocks
if db.is_pruned() && self.chain.best_block_number() >= block_number + HISTORY {
if db.is_pruned() && self.chain.read().best_block_number() >= block_number + HISTORY {
return None;
}
@ -522,7 +529,7 @@ impl Client {
/// Get a copy of the best block's state.
pub fn state(&self) -> State {
State::from_existing(
self.state_db.lock().boxed_clone(),
self.state_db.read().boxed_clone(),
HeaderView::new(&self.best_block_header()).state_root(),
self.engine.account_start_nonce(),
self.factories.clone())
@ -531,22 +538,22 @@ impl Client {
/// Get info on the cache.
pub fn blockchain_cache_info(&self) -> BlockChainCacheSize {
self.chain.cache_size()
self.chain.read().cache_size()
}
/// Get the report.
pub fn report(&self) -> ClientReport {
let mut report = self.report.read().clone();
report.state_db_mem = self.state_db.lock().mem_used();
report.state_db_mem = self.state_db.read().mem_used();
report
}
/// Tick the client.
// TODO: manage by real events.
pub fn tick(&self) {
self.chain.collect_garbage();
self.chain.read().collect_garbage();
self.block_queue.collect_garbage();
self.tracedb.collect_garbage();
self.tracedb.read().collect_garbage();
match self.mode {
Mode::Dark(timeout) => {
@ -584,16 +591,16 @@ impl Client {
pub fn block_number(&self, id: BlockID) -> Option<BlockNumber> {
match id {
BlockID::Number(number) => Some(number),
BlockID::Hash(ref hash) => self.chain.block_number(hash),
BlockID::Hash(ref hash) => self.chain.read().block_number(hash),
BlockID::Earliest => Some(0),
BlockID::Latest | BlockID::Pending => Some(self.chain.best_block_number()),
BlockID::Latest | BlockID::Pending => Some(self.chain.read().best_block_number()),
}
}
/// Take a snapshot at the given block.
/// If the ID given is "latest", this will default to 1000 blocks behind.
pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), ::error::Error> {
let db = self.state_db.lock().boxed_clone();
pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), EthcoreError> {
let db = self.state_db.read().boxed_clone();
let best_block_number = self.chain_info().best_block_number;
let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at)));
@ -618,7 +625,7 @@ impl Client {
},
};
try!(snapshot::take_snapshot(&self.chain, start_hash, db.as_hashdb(), writer, p));
try!(snapshot::take_snapshot(&self.chain.read(), start_hash, db.as_hashdb(), writer, p));
Ok(())
}
@ -634,8 +641,8 @@ impl Client {
fn transaction_address(&self, id: TransactionID) -> Option<TransactionAddress> {
match id {
TransactionID::Hash(ref hash) => self.chain.transaction_address(hash),
TransactionID::Location(id, index) => Self::block_hash(&self.chain, id).map(|hash| TransactionAddress {
TransactionID::Hash(ref hash) => self.chain.read().transaction_address(hash),
TransactionID::Location(id, index) => Self::block_hash(&self.chain.read(), id).map(|hash| TransactionAddress {
block_hash: hash,
index: index,
})
@ -666,6 +673,25 @@ impl Client {
}
}
impl snapshot::DatabaseRestore for Client {
/// Restart the client with a new backend
fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError> {
let _import_lock = self.import_lock.lock();
let mut state_db = self.state_db.write();
let mut chain = self.chain.write();
let mut tracedb = self.tracedb.write();
self.miner.clear();
let db = self.db.write();
try!(db.restore(new_db));
*state_db = journaldb::new(db.clone(), self.pruning, ::db::COL_STATE);
*chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone()));
*tracedb = try!(TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone()).map_err(ClientError::from));
Ok(())
}
}
impl BlockChainClient for Client {
fn call(&self, t: &SignedTransaction, block: BlockID, analytics: CallAnalytics) -> Result<Executed, CallError> {
let header = try!(self.block_header(block).ok_or(CallError::StatePruned));
@ -749,15 +775,17 @@ impl BlockChainClient for Client {
}
fn best_block_header(&self) -> Bytes {
self.chain.best_block_header()
self.chain.read().best_block_header()
}
fn block_header(&self, id: BlockID) -> Option<Bytes> {
Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_header_data(&hash))
let chain = self.chain.read();
Self::block_hash(&chain, id).and_then(|hash| chain.block_header_data(&hash))
}
fn block_body(&self, id: BlockID) -> Option<Bytes> {
Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_body(&hash))
let chain = self.chain.read();
Self::block_hash(&chain, id).and_then(|hash| chain.block_body(&hash))
}
fn block(&self, id: BlockID) -> Option<Bytes> {
@ -766,14 +794,16 @@ impl BlockChainClient for Client {
return Some(block.rlp_bytes(Seal::Without));
}
}
Self::block_hash(&self.chain, id).and_then(|hash| {
self.chain.block(&hash)
let chain = self.chain.read();
Self::block_hash(&chain, id).and_then(|hash| {
chain.block(&hash)
})
}
fn block_status(&self, id: BlockID) -> BlockStatus {
match Self::block_hash(&self.chain, id) {
Some(ref hash) if self.chain.is_known(hash) => BlockStatus::InChain,
let chain = self.chain.read();
match Self::block_hash(&chain, id) {
Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain,
Some(hash) => self.block_queue.block_status(&hash),
None => BlockStatus::Unknown
}
@ -785,7 +815,8 @@ impl BlockChainClient for Client {
return Some(*block.header.difficulty() + self.block_total_difficulty(BlockID::Latest).expect("blocks in chain have details; qed"));
}
}
Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_details(&hash)).map(|d| d.total_difficulty)
let chain = self.chain.read();
Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty)
}
fn nonce(&self, address: &Address, id: BlockID) -> Option<U256> {
@ -793,7 +824,8 @@ impl BlockChainClient for Client {
}
fn block_hash(&self, id: BlockID) -> Option<H256> {
Self::block_hash(&self.chain, id)
let chain = self.chain.read();
Self::block_hash(&chain, id)
}
fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>> {
@ -809,7 +841,7 @@ impl BlockChainClient for Client {
}
fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction> {
self.transaction_address(id).and_then(|address| self.chain.transaction(&address))
self.transaction_address(id).and_then(|address| self.chain.read().transaction(&address))
}
fn uncle(&self, id: UncleID) -> Option<Bytes> {
@ -818,11 +850,12 @@ impl BlockChainClient for Client {
}
fn transaction_receipt(&self, id: TransactionID) -> Option<LocalizedReceipt> {
self.transaction_address(id).and_then(|address| self.chain.block_number(&address.block_hash).and_then(|block_number| {
let t = self.chain.block_body(&address.block_hash)
let chain = self.chain.read();
self.transaction_address(id).and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| {
let t = chain.block_body(&address.block_hash)
.and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index));
match (t, self.chain.transaction_receipt(&address)) {
match (t, chain.transaction_receipt(&address)) {
(Some(tx), Some(receipt)) => {
let block_hash = tx.block_hash.clone();
let block_number = tx.block_number.clone();
@ -832,7 +865,7 @@ impl BlockChainClient for Client {
0 => U256::zero(),
i => {
let prior_address = TransactionAddress { block_hash: address.block_hash, index: i - 1 };
let prior_receipt = self.chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed");
let prior_receipt = chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed");
prior_receipt.gas_used
}
};
@ -863,28 +896,29 @@ impl BlockChainClient for Client {
}
fn tree_route(&self, from: &H256, to: &H256) -> Option<TreeRoute> {
match self.chain.is_known(from) && self.chain.is_known(to) {
true => Some(self.chain.tree_route(from.clone(), to.clone())),
let chain = self.chain.read();
match chain.is_known(from) && chain.is_known(to) {
true => Some(chain.tree_route(from.clone(), to.clone())),
false => None
}
}
fn find_uncles(&self, hash: &H256) -> Option<Vec<H256>> {
self.chain.find_uncle_hashes(hash, self.engine.maximum_uncle_age())
self.chain.read().find_uncle_hashes(hash, self.engine.maximum_uncle_age())
}
fn state_data(&self, hash: &H256) -> Option<Bytes> {
self.state_db.lock().state(hash)
self.state_db.read().state(hash)
}
fn block_receipts(&self, hash: &H256) -> Option<Bytes> {
self.chain.block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec())
self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec())
}
fn import_block(&self, bytes: Bytes) -> Result<H256, BlockImportError> {
{
let header = BlockView::new(&bytes).header_view();
if self.chain.is_known(&header.sha3()) {
if self.chain.read().is_known(&header.sha3()) {
return Err(BlockImportError::Import(ImportError::AlreadyInChain));
}
if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown {
@ -903,12 +937,13 @@ impl BlockChainClient for Client {
}
fn chain_info(&self) -> BlockChainInfo {
let chain = self.chain.read();
BlockChainInfo {
total_difficulty: self.chain.best_block_total_difficulty(),
pending_total_difficulty: self.chain.best_block_total_difficulty(),
genesis_hash: self.chain.genesis_hash(),
best_block_hash: self.chain.best_block_hash(),
best_block_number: From::from(self.chain.best_block_number())
total_difficulty: chain.best_block_total_difficulty(),
pending_total_difficulty: chain.best_block_total_difficulty(),
genesis_hash: chain.genesis_hash(),
best_block_hash: chain.best_block_hash(),
best_block_number: From::from(chain.best_block_number())
}
}
@ -918,7 +953,7 @@ impl BlockChainClient for Client {
fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockID, to_block: BlockID) -> Option<Vec<BlockNumber>> {
match (self.block_number(from_block), self.block_number(to_block)) {
(Some(from), Some(to)) => Some(self.chain.blocks_with_bloom(bloom, from, to)),
(Some(from), Some(to)) => Some(self.chain.read().blocks_with_bloom(bloom, from, to)),
_ => None
}
}
@ -936,10 +971,11 @@ impl BlockChainClient for Client {
blocks.sort();
let chain = self.chain.read();
blocks.into_iter()
.filter_map(|number| self.chain.block_hash(number).map(|hash| (number, hash)))
.filter_map(|(number, hash)| self.chain.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
.filter_map(|(number, hash, receipts)| self.chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes())))
.filter_map(|number| chain.block_hash(number).map(|hash| (number, hash)))
.filter_map(|(number, hash)| chain.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
.filter_map(|(number, hash, receipts)| chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes())))
.flat_map(|(number, hash, receipts, hashes)| {
let mut log_index = 0;
receipts.into_iter()
@ -975,7 +1011,7 @@ impl BlockChainClient for Client {
to_address: From::from(filter.to_address),
};
let traces = self.tracedb.filter(&filter);
let traces = self.tracedb.read().filter(&filter);
Some(traces)
} else {
None
@ -987,7 +1023,7 @@ impl BlockChainClient for Client {
self.transaction_address(trace.transaction)
.and_then(|tx_address| {
self.block_number(BlockID::Hash(tx_address.block_hash))
.and_then(|number| self.tracedb.trace(number, tx_address.index, trace_address))
.and_then(|number| self.tracedb.read().trace(number, tx_address.index, trace_address))
})
}
@ -995,17 +1031,17 @@ impl BlockChainClient for Client {
self.transaction_address(transaction)
.and_then(|tx_address| {
self.block_number(BlockID::Hash(tx_address.block_hash))
.and_then(|number| self.tracedb.transaction_traces(number, tx_address.index))
.and_then(|number| self.tracedb.read().transaction_traces(number, tx_address.index))
})
}
fn block_traces(&self, block: BlockID) -> Option<Vec<LocalizedTrace>> {
self.block_number(block)
.and_then(|number| self.tracedb.block_traces(number))
.and_then(|number| self.tracedb.read().block_traces(number))
}
fn last_hashes(&self) -> LastHashes {
(*self.build_last_hashes(self.chain.best_block_hash())).clone()
(*self.build_last_hashes(self.chain.read().best_block_hash())).clone()
}
fn queue_transactions(&self, transactions: Vec<Bytes>) {
@ -1032,14 +1068,15 @@ impl BlockChainClient for Client {
impl MiningBlockChainClient for Client {
fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock {
let engine = &*self.engine;
let h = self.chain.best_block_hash();
let chain = self.chain.read();
let h = chain.best_block_hash();
let mut open_block = OpenBlock::new(
engine,
self.factories.clone(),
false, // TODO: this will need to be parameterised once we want to do immediate mining insertion.
self.state_db.lock().boxed_clone(),
&self.chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
self.state_db.read().boxed_clone(),
&chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
self.build_last_hashes(h.clone()),
author,
gas_range_target,
@ -1047,7 +1084,7 @@ impl MiningBlockChainClient for Client {
).expect("OpenBlock::new only fails if parent state root invalid; state root of best block's header is never invalid; qed");
// Add uncles
self.chain
chain
.find_uncle_headers(&h, engine.maximum_uncle_age())
.unwrap()
.into_iter()
@ -1088,7 +1125,7 @@ impl MiningBlockChainClient for Client {
precise_time_ns() - start,
);
});
self.db.flush().expect("DB flush failed.");
self.db.read().flush().expect("DB flush failed.");
Ok(h)
}
}

2
ethcore/src/client/config.rs
View File

@ -62,7 +62,7 @@ impl FromStr for DatabaseCompactionProfile {
}
/// Operating mode for the client.
#[derive(Debug, Eq, PartialEq)]
#[derive(Debug, Eq, PartialEq, Clone)]
pub enum Mode {
/// Always on.
Active,

5
ethcore/src/miner/miner.rs
View File

@ -227,6 +227,11 @@ impl Miner {
self.options.force_sealing || !self.options.new_work_notify.is_empty()
}
/// Clear all pending block states
pub fn clear(&self) {
self.sealing_work.lock().queue.reset();
}
/// Get `Some` `clone()` of the current pending block's state or `None` if we're not sealing.
pub fn pending_state(&self) -> Option<State> {
self.sealing_work.lock().queue.peek_last_ref().map(|b| b.block().fields().state.clone())

38
ethcore/src/service.rs
View File

@ -46,6 +46,8 @@ pub enum ClientIoMessage {
FeedStateChunk(H256, Bytes),
/// Feed a block chunk to the snapshot service
FeedBlockChunk(H256, Bytes),
/// Take a snapshot for the block with given number.
TakeSnapshot(u64),
}
/// Client service setup. Creates and registers client and network services with the IO subsystem.
@ -78,7 +80,7 @@ impl ClientService {
let pruning = config.pruning;
let client = try!(Client::new(config, &spec, db_path, miner, io_service.channel()));
let snapshot = try!(SnapshotService::new(spec, pruning, db_path.into(), io_service.channel()));
let snapshot = try!(SnapshotService::new(spec, pruning, db_path.into(), io_service.channel(), client.clone()));
let snapshot = Arc::new(snapshot);
@ -90,7 +92,7 @@ impl ClientService {
try!(io_service.register_handler(client_io));
let stop_guard = ::devtools::StopGuard::new();
run_ipc(ipc_path, client.clone(), stop_guard.share());
run_ipc(ipc_path, client.clone(), snapshot.clone(), stop_guard.share());
Ok(ClientService {
io_service: Arc::new(io_service),
@ -145,16 +147,22 @@ struct ClientIoHandler {
}
const CLIENT_TICK_TIMER: TimerToken = 0;
const SNAPSHOT_TICK_TIMER: TimerToken = 1;
const CLIENT_TICK_MS: u64 = 5000;
const SNAPSHOT_TICK_MS: u64 = 10000;
impl IoHandler<ClientIoMessage> for ClientIoHandler {
fn initialize(&self, io: &IoContext<ClientIoMessage>) {
io.register_timer(CLIENT_TICK_TIMER, CLIENT_TICK_MS).expect("Error registering client timer");
io.register_timer(SNAPSHOT_TICK_TIMER, SNAPSHOT_TICK_MS).expect("Error registering snapshot timer");
}
fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
if timer == CLIENT_TICK_TIMER {
self.client.tick();
match timer {
CLIENT_TICK_TIMER => self.client.tick(),
SNAPSHOT_TICK_TIMER => self.snapshot.tick(),
_ => warn!("IO service triggered unregistered timer '{}'", timer),
}
}
@ -170,20 +178,38 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
}
ClientIoMessage::FeedStateChunk(ref hash, ref chunk) => self.snapshot.feed_state_chunk(*hash, chunk),
ClientIoMessage::FeedBlockChunk(ref hash, ref chunk) => self.snapshot.feed_block_chunk(*hash, chunk),
ClientIoMessage::TakeSnapshot(num) => {
if let Err(e) = self.snapshot.take_snapshot(&*self.client, num) {
warn!("Failed to take snapshot at block #{}: {}", num, e);
}
}
_ => {} // ignore other messages
}
}
}
#[cfg(feature="ipc")]
fn run_ipc(base_path: &Path, client: Arc<Client>, stop: Arc<AtomicBool>) {
fn run_ipc(base_path: &Path, client: Arc<Client>, snapshot_service: Arc<SnapshotService>, stop: Arc<AtomicBool>) {
let mut path = base_path.to_owned();
path.push("parity-chain.ipc");
let socket_addr = format!("ipc://{}", path.to_string_lossy());
let s = stop.clone();
::std::thread::spawn(move || {
let mut worker = nanoipc::Worker::new(&(client as Arc<BlockChainClient>));
worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues");
while !s.load(::std::sync::atomic::Ordering::Relaxed) {
worker.poll();
}
});
let mut path = base_path.to_owned();
path.push("parity-snapshot.ipc");
let socket_addr = format!("ipc://{}", path.to_string_lossy());
::std::thread::spawn(move || {
let mut worker = nanoipc::Worker::new(&(snapshot_service as Arc<::snapshot::SnapshotService>));
worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues");
while !stop.load(::std::sync::atomic::Ordering::Relaxed) {
worker.poll();
}
@ -191,7 +217,7 @@ fn run_ipc(base_path: &Path, client: Arc<Client>, stop: Arc<AtomicBool>) {
}
#[cfg(not(feature="ipc"))]
fn run_ipc(_base_path: &Path, _client: Arc<Client>, _stop: Arc<AtomicBool>) {
fn run_ipc(_base_path: &Path, _client: Arc<Client>, _snapshot_service: Arc<SnapshotService>, _stop: Arc<AtomicBool>) {
}
#[cfg(test)]

86
ethcore/src/snapshot/mod.rs
View File

@ -32,9 +32,9 @@ use util::Mutex;
use util::hash::{FixedHash, H256};
use util::journaldb::{self, Algorithm, JournalDB};
use util::kvdb::Database;
use util::sha3::SHA3_NULL_RLP;
use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
use util::sha3::SHA3_NULL_RLP;
use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
use self::account::Account;
use self::block::AbridgedBlock;
@ -44,7 +44,12 @@ use crossbeam::{scope, ScopedJoinHandle};
use rand::{Rng, OsRng};
pub use self::error::Error;
pub use self::service::{RestorationStatus, Service, SnapshotService};
pub use self::service::{Service, DatabaseRestore};
pub use self::traits::{SnapshotService, RemoteSnapshotService};
pub use self::watcher::Watcher;
pub use types::snapshot_manifest::ManifestData;
pub use types::restoration_status::RestorationStatus;
pub mod io;
pub mod service;
@ -52,10 +57,16 @@ pub mod service;
mod account;
mod block;
mod error;
mod watcher;
#[cfg(test)]
mod tests;
mod traits {
#![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues
include!(concat!(env!("OUT_DIR"), "/snapshot_service_trait.rs"));
}
// Try to have chunks be around 4MB (before compression)
const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
@ -72,17 +83,28 @@ pub struct Progress {
}
impl Progress {
/// Reset the progress.
pub fn reset(&self) {
self.accounts.store(0, Ordering::Release);
self.blocks.store(0, Ordering::Release);
self.size.store(0, Ordering::Release);
// atomic fence here to ensure the others are written first?
// logs might very rarely get polluted if not.
self.done.store(false, Ordering::Release);
}
/// Get the number of accounts snapshotted thus far.
pub fn accounts(&self) -> usize { self.accounts.load(Ordering::Relaxed) }
pub fn accounts(&self) -> usize { self.accounts.load(Ordering::Acquire) }
/// Get the number of blocks snapshotted thus far.
pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Relaxed) }
pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) }
/// Get the written size of the snapshot in bytes.
pub fn size(&self) -> usize { self.size.load(Ordering::Relaxed) }
pub fn size(&self) -> usize { self.size.load(Ordering::Acquire) }
/// Whether the snapshot is complete.
pub fn done(&self) -> bool { self.done.load(Ordering::SeqCst) }
pub fn done(&self) -> bool { self.done.load(Ordering::Acquire) }
}
/// Take a snapshot using the given blockchain, starting block hash, and database, writing into the given writer.
@ -354,54 +376,6 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
Ok(chunker.hashes)
}
/// Manifest data.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ManifestData {
/// List of state chunk hashes.
pub state_hashes: Vec<H256>,
/// List of block chunk hashes.
pub block_hashes: Vec<H256>,
/// The final, expected state root.
pub state_root: H256,
/// Block number this snapshot was taken at.
pub block_number: u64,
/// Block hash this snapshot was taken at.
pub block_hash: H256,
}
impl ManifestData {
/// Encode the manifest data to rlp.
pub fn into_rlp(self) -> Bytes {
let mut stream = RlpStream::new_list(5);
stream.append(&self.state_hashes);
stream.append(&self.block_hashes);
stream.append(&self.state_root);
stream.append(&self.block_number);
stream.append(&self.block_hash);
stream.out()
}
/// Try to restore manifest data from raw bytes, interpreted as RLP.
pub fn from_rlp(raw: &[u8]) -> Result<Self, DecoderError> {
let decoder = UntrustedRlp::new(raw);
let state_hashes: Vec<H256> = try!(decoder.val_at(0));
let block_hashes: Vec<H256> = try!(decoder.val_at(1));
let state_root: H256 = try!(decoder.val_at(2));
let block_number: u64 = try!(decoder.val_at(3));
let block_hash: H256 = try!(decoder.val_at(4));
Ok(ManifestData {
state_hashes: state_hashes,
block_hashes: block_hashes,
state_root: state_root,
block_number: block_number,
block_hash: block_hash,
})
}
}
/// Used to rebuild the state trie piece by piece.
pub struct StateRebuilder {
db: Box<JournalDB>,
@ -653,4 +627,4 @@ impl BlockRebuilder {
}
}
}
}
}

198
ethcore/src/snapshot/service.rs
View File

@ -23,12 +23,14 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use super::{ManifestData, StateRebuilder, BlockRebuilder};
use super::{ManifestData, StateRebuilder, BlockRebuilder, RestorationStatus, SnapshotService};
use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
use blockchain::BlockChain;
use client::Client;
use engines::Engine;
use error::Error;
use ids::BlockID;
use service::ClientIoMessage;
use spec::Spec;
@ -39,51 +41,27 @@ use util::journaldb::Algorithm;
use util::kvdb::{Database, DatabaseConfig};
use util::snappy;
/// Statuses for restorations.
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub enum RestorationStatus {
/// No restoration.
Inactive,
/// Ongoing restoration.
Ongoing,
/// Failed restoration.
Failed,
/// Helper for removing directories in case of error.
struct Guard(bool, PathBuf);
impl Guard {
fn new(path: PathBuf) -> Self { Guard(true, path) }
fn disarm(mut self) { self.0 = false }
}
/// The interface for a snapshot network service.
/// This handles:
/// - restoration of snapshots to temporary databases.
/// - responding to queries for snapshot manifests and chunks
pub trait SnapshotService {
/// Query the most recent manifest data.
fn manifest(&self) -> Option<ManifestData>;
impl Drop for Guard {
fn drop(&mut self) {
if self.0 {
let _ = fs::remove_dir_all(&self.1);
}
}
}
/// Get raw chunk for a given hash.
fn chunk(&self, hash: H256) -> Option<Bytes>;
/// Ask the snapshot service for the restoration status.
fn status(&self) -> RestorationStatus;
/// Ask the snapshot service for the number of chunks completed.
/// Return a tuple of (state_chunks, block_chunks).
/// Undefined when not restoring.
fn chunks_done(&self) -> (usize, usize);
/// Begin snapshot restoration.
/// If restoration in-progress, this will reset it.
/// From this point on, any previous snapshot may become unavailable.
fn begin_restore(&self, manifest: ManifestData);
/// Abort an in-progress restoration if there is one.
fn abort_restore(&self);
/// Feed a raw state chunk to the service to be processed asynchronously.
/// no-op if not currently restoring.
fn restore_state_chunk(&self, hash: H256, chunk: Bytes);
/// Feed a raw block chunk to the service to be processed asynchronously.
/// no-op if currently restoring.
fn restore_block_chunk(&self, hash: H256, chunk: Bytes);
/// External database restoration handler
pub trait DatabaseRestore: Send + Sync {
/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
fn restore_db(&self, new_db: &str) -> Result<(), Error>;
}
/// State restoration manager.
@ -96,6 +74,7 @@ struct Restoration {
writer: LooseWriter,
snappy_buffer: Bytes,
final_state_root: H256,
guard: Guard,
}
struct RestorationParams<'a> {
@ -104,6 +83,7 @@ struct RestorationParams<'a> {
db_path: PathBuf, // database path
writer: LooseWriter, // writer for recovered snapshot.
genesis: &'a [u8], // genesis block of the chain.
guard: Guard, // guard for the restoration directory.
}
impl Restoration {
@ -131,6 +111,7 @@ impl Restoration {
writer: params.writer,
snappy_buffer: Vec::new(),
final_state_root: root,
guard: params.guard,
})
}
@ -179,6 +160,7 @@ impl Restoration {
try!(self.writer.finish(self.manifest));
self.guard.disarm();
Ok(())
}
@ -208,11 +190,13 @@ pub struct Service {
genesis_block: Bytes,
state_chunks: AtomicUsize,
block_chunks: AtomicUsize,
db_restore: Arc<DatabaseRestore>,
progress: super::Progress,
}
impl Service {
/// Create a new snapshot service.
pub fn new(spec: &Spec, pruning: Algorithm, client_db: PathBuf, io_channel: Channel) -> Result<Self, Error> {
pub fn new(spec: &Spec, pruning: Algorithm, client_db: PathBuf, io_channel: Channel, db_restore: Arc<DatabaseRestore>) -> Result<Self, Error> {
let db_path = try!(client_db.parent().and_then(Path::parent)
.ok_or_else(|| UtilError::SimpleString("Failed to find database root.".into()))).to_owned();
@ -236,6 +220,8 @@ impl Service {
genesis_block: spec.genesis_block(),
state_chunks: AtomicUsize::new(0),
block_chunks: AtomicUsize::new(0),
db_restore: db_restore,
progress: Default::default(),
};
// create the root snapshot dir if it doesn't exist.
@ -252,6 +238,13 @@ impl Service {
}
}
// delete the temporary snapshot dir if it does exist.
if let Err(e) = fs::remove_dir_all(service.temp_snapshot_dir()) {
if e.kind() != ErrorKind::NotFound {
return Err(e.into())
}
}
Ok(service)
}
@ -269,6 +262,13 @@ impl Service {
dir
}
// get the temporary snapshot dir.
fn temp_snapshot_dir(&self) -> PathBuf {
let mut dir = self.root_dir();
dir.push("in_progress");
dir
}
// get the restoration directory.
fn restoration_dir(&self) -> PathBuf {
let mut dir = self.root_dir();
@ -295,37 +295,50 @@ impl Service {
let our_db = self.restoration_db();
trace!(target: "snapshot", "replacing {:?} with {:?}", self.client_db, our_db);
try!(self.db_restore.restore_db(our_db.to_str().unwrap()));
Ok(())
}
let mut backup_db = self.restoration_dir();
backup_db.push("backup_db");
/// Tick the snapshot service. This will log any active snapshot
/// being taken.
pub fn tick(&self) {
if self.progress.done() { return }
let _ = fs::remove_dir_all(&backup_db);
let p = &self.progress;
info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size());
}
let existed = match fs::rename(&self.client_db, &backup_db) {
Ok(_) => true,
Err(e) => if let ErrorKind::NotFound = e.kind() {
false
} else {
return Err(e.into());
}
};
/// Take a snapshot at the block with the given number.
/// calling this while a restoration is in progress or vice versa
/// will lead to a race condition where the first one to finish will
/// have their produced snapshot overwritten.
pub fn take_snapshot(&self, client: &Client, num: u64) -> Result<(), Error> {
info!("Taking snapshot at #{}", num);
self.progress.reset();
match fs::rename(&our_db, &self.client_db) {
Ok(_) => {
// clean up the backup.
if existed {
try!(fs::remove_dir_all(&backup_db));
}
Ok(())
}
Err(e) => {
// restore the backup.
if existed {
try!(fs::rename(&backup_db, &self.client_db));
}
Err(e.into())
}
}
let temp_dir = self.temp_snapshot_dir();
let snapshot_dir = self.snapshot_dir();
let _ = fs::remove_dir_all(&temp_dir);
let writer = try!(LooseWriter::new(temp_dir.clone()));
let guard = Guard::new(temp_dir.clone());
try!(client.take_snapshot(writer, BlockID::Number(num), &self.progress));
info!("Finished taking snapshot at #{}", num);
let mut reader = self.reader.write();
// destroy the old snapshot reader.
*reader = None;
try!(fs::rename(temp_dir, &snapshot_dir));
*reader = Some(try!(LooseReader::new(snapshot_dir)));
guard.disarm();
Ok(())
}
/// Initialize the restoration synchronously.
@ -356,11 +369,15 @@ impl Service {
db_path: self.restoration_db(),
writer: writer,
genesis: &self.genesis_block,
guard: Guard::new(rest_dir),
};
*res = Some(try!(Restoration::new(params)));
*self.status.lock() = RestorationStatus::Ongoing;
*self.status.lock() = RestorationStatus::Ongoing {
state_chunks_done: self.state_chunks.load(Ordering::Relaxed) as u32,
block_chunks_done: self.block_chunks.load(Ordering::Relaxed) as u32,
};
Ok(())
}
@ -393,14 +410,7 @@ impl Service {
try!(fs::create_dir(&snapshot_dir));
trace!(target: "snapshot", "copying restored snapshot files over");
for maybe_file in try!(fs::read_dir(self.temp_recovery_dir())) {
let path = try!(maybe_file).path();
if let Some(name) = path.file_name().map(|x| x.to_owned()) {
let mut new_path = snapshot_dir.clone();
new_path.push(name);
try!(fs::rename(path, new_path));
}
}
try!(fs::rename(self.temp_recovery_dir(), &snapshot_dir));
let _ = fs::remove_dir_all(self.restoration_dir());
@ -418,7 +428,7 @@ impl Service {
match self.status() {
RestorationStatus::Inactive | RestorationStatus::Failed => Ok(()),
RestorationStatus::Ongoing => {
RestorationStatus::Ongoing { .. } => {
let res = {
let rest = match *restoration {
Some(ref mut r) => r,
@ -489,10 +499,6 @@ impl SnapshotService for Service {
*self.status.lock()
}
fn chunks_done(&self) -> (usize, usize) {
(self.state_chunks.load(Ordering::Relaxed), self.block_chunks.load(Ordering::Relaxed))
}
fn begin_restore(&self, manifest: ManifestData) {
self.io_channel.send(ClientIoMessage::BeginRestoration(manifest))
.expect("snapshot service and io service are kept alive by client service; qed");
@ -520,17 +526,31 @@ impl SnapshotService for Service {
}
}
impl Drop for Service {
fn drop(&mut self) {
self.abort_restore();
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use service::ClientIoMessage;
use io::{IoService};
use devtools::RandomTempPath;
use tests::helpers::get_test_spec;
use util::journaldb::Algorithm;
use snapshot::ManifestData;
use error::Error;
use snapshot::{ManifestData, RestorationStatus, SnapshotService};
use super::*;
struct NoopDBRestore;
impl DatabaseRestore for NoopDBRestore {
fn restore_db(&self, _new_db: &str) -> Result<(), Error> {
Ok(())
}
}
#[test]
fn sends_async_messages() {
let service = IoService::<ClientIoMessage>::start().unwrap();
@ -544,13 +564,13 @@ mod tests {
&get_test_spec(),
Algorithm::Archive,
dir,
service.channel()
service.channel(),
Arc::new(NoopDBRestore),
).unwrap();
assert!(service.manifest().is_none());
assert!(service.chunk(Default::default()).is_none());
assert_eq!(service.status(), RestorationStatus::Inactive);
assert_eq!(service.chunks_done(), (0, 0));
let manifest = ManifestData {
state_hashes: vec![],

54
ethcore/src/snapshot/snapshot_service_trait.rs Normal file
View File

@ -0,0 +1,54 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use super::{ManifestData, RestorationStatus};
use util::{Bytes, H256};
use ipc::IpcConfig;
/// The interface for a snapshot network service.
/// This handles:
/// - restoration of snapshots to temporary databases.
/// - responding to queries for snapshot manifests and chunks
#[derive(Ipc)]
#[ipc(client_ident="RemoteSnapshotService")]
pub trait SnapshotService : Sync + Send {
/// Query the most recent manifest data.
fn manifest(&self) -> Option<ManifestData>;
/// Get raw chunk for a given hash.
fn chunk(&self, hash: H256) -> Option<Bytes>;
/// Ask the snapshot service for the restoration status.
fn status(&self) -> RestorationStatus;
/// Begin snapshot restoration.
/// If restoration in-progress, this will reset it.
/// From this point on, any previous snapshot may become unavailable.
fn begin_restore(&self, manifest: ManifestData);
/// Abort an in-progress restoration if there is one.
fn abort_restore(&self);
/// Feed a raw state chunk to the service to be processed asynchronously.
/// no-op if not currently restoring.
fn restore_state_chunk(&self, hash: H256, chunk: Bytes);
/// Feed a raw block chunk to the service to be processed asynchronously.
/// no-op if currently restoring.
fn restore_block_chunk(&self, hash: H256, chunk: Bytes);
}
impl IpcConfig for SnapshotService { }

192
ethcore/src/snapshot/watcher.rs Normal file
View File

@ -0,0 +1,192 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Watcher for snapshot-related chain events.
use client::{BlockChainClient, Client, ChainNotify};
use ids::BlockID;
use service::ClientIoMessage;
use views::HeaderView;
use io::IoChannel;
use util::hash::H256;
use std::sync::Arc;
// helper trait for transforming hashes to numbers and checking if syncing.
trait Oracle: Send + Sync {
fn to_number(&self, hash: H256) -> Option<u64>;
fn is_major_syncing(&self) -> bool;
}
impl Oracle for Client {
fn to_number(&self, hash: H256) -> Option<u64> {
self.block_header(BlockID::Hash(hash)).map(|h| HeaderView::new(&h).number())
}
fn is_major_syncing(&self) -> bool {
let queue_info = self.queue_info();
queue_info.unverified_queue_size + queue_info.verified_queue_size > 3
}
}
// helper trait for broadcasting a block to take a snapshot at.
trait Broadcast: Send + Sync {
fn take_at(&self, num: Option<u64>);
}
impl Broadcast for IoChannel<ClientIoMessage> {
fn take_at(&self, num: Option<u64>) {
let num = match num {
Some(n) => n,
None => return,
};
trace!(target: "snapshot_watcher", "broadcast: {}", num);
if let Err(e) = self.send(ClientIoMessage::TakeSnapshot(num)) {
warn!("Snapshot watcher disconnected from IoService: {}", e);
}
}
}
/// A `ChainNotify` implementation which will trigger a snapshot event
/// at certain block numbers.
pub struct Watcher {
oracle: Arc<Oracle>,
broadcast: Box<Broadcast>,
period: u64,
history: u64,
}
impl Watcher {
/// Create a new `Watcher` which will trigger a snapshot event
/// once every `period` blocks, but only after that block is
/// `history` blocks old.
pub fn new(client: Arc<Client>, channel: IoChannel<ClientIoMessage>, period: u64, history: u64) -> Self {
Watcher {
oracle: client,
broadcast: Box::new(channel),
period: period,
history: history,
}
}
}
impl ChainNotify for Watcher {
fn new_blocks(
&self,
imported: Vec<H256>,
_: Vec<H256>,
_: Vec<H256>,
_: Vec<H256>,
_: Vec<H256>,
_duration: u64)
{
if self.oracle.is_major_syncing() { return }
trace!(target: "snapshot_watcher", "{} imported", imported.len());
let highest = imported.into_iter()
.filter_map(|h| self.oracle.to_number(h))
.filter(|&num| num >= self.period + self.history)
.map(|num| num - self.history)
.filter(|num| num % self.period == 0)
.fold(0, ::std::cmp::max);
match highest {
0 => self.broadcast.take_at(None),
_ => self.broadcast.take_at(Some(highest)),
}
}
}
#[cfg(test)]
mod tests {
use super::{Broadcast, Oracle, Watcher};
use client::ChainNotify;
use util::{H256, U256};
use std::collections::HashMap;
use std::sync::Arc;
struct TestOracle(HashMap<H256, u64>);
impl Oracle for TestOracle {
fn to_number(&self, hash: H256) -> Option<u64> {
self.0.get(&hash).cloned()
}
fn is_major_syncing(&self) -> bool { false }
}
struct TestBroadcast(Option<u64>);
impl Broadcast for TestBroadcast {
fn take_at(&self, num: Option<u64>) {
if num != self.0 {
panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num);
}
}
}
// helper harness for tests which expect a notification.
fn harness(numbers: Vec<u64>, period: u64, history: u64, expected: Option<u64>) {
let hashes: Vec<_> = numbers.clone().into_iter().map(|x| H256::from(U256::from(x))).collect();
let map = hashes.clone().into_iter().zip(numbers).collect();
let watcher = Watcher {
oracle: Arc::new(TestOracle(map)),
broadcast: Box::new(TestBroadcast(expected)),
period: period,
history: history,
};
watcher.new_blocks(
hashes,
vec![],
vec![],
vec![],
vec![],
0,
);
}
// helper
#[test]
fn should_not_fire() {
harness(vec![0], 5, 0, None);
}
#[test]
fn fires_once_for_two() {
harness(vec![14, 15], 10, 5, Some(10));
}
#[test]
fn finds_highest() {
harness(vec![15, 25], 10, 5, Some(20));
}
#[test]
fn doesnt_fire_before_history() {
harness(vec![10, 11], 10, 5, None);
}
}

2
ethcore/src/types/mod.rs.in
View File

@ -31,3 +31,5 @@ pub mod trace_filter;
pub mod call_analytics;
pub mod transaction_import;
pub mod block_import_error;
pub mod restoration_status;
pub mod snapshot_manifest;

34
ethcore/src/types/restoration_status.rs Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Restoration status type definition
/// Statuses for restorations.
#[derive(PartialEq, Eq, Clone, Copy, Debug, Binary)]
pub enum RestorationStatus {
/// No restoration.
Inactive,
/// Ongoing restoration.
Ongoing {
/// Number of state chunks completed.
state_chunks_done: u32,
/// Number of block chunks completed.
block_chunks_done: u32,
},
/// Failed restoration.
Failed,
}

70
ethcore/src/types/snapshot_manifest.rs Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Snapshot manifest type definition
use util::hash::H256;
use rlp::*;
use util::Bytes;
/// Manifest data.
#[derive(Debug, Clone, PartialEq, Eq, Binary)]
pub struct ManifestData {
/// List of state chunk hashes.
pub state_hashes: Vec<H256>,
/// List of block chunk hashes.
pub block_hashes: Vec<H256>,
/// The final, expected state root.
pub state_root: H256,
/// Block number this snapshot was taken at.
pub block_number: u64,
/// Block hash this snapshot was taken at.
pub block_hash: H256,
}
impl ManifestData {
/// Encode the manifest data to rlp.
pub fn into_rlp(self) -> Bytes {
let mut stream = RlpStream::new_list(5);
stream.append(&self.state_hashes);
stream.append(&self.block_hashes);
stream.append(&self.state_root);
stream.append(&self.block_number);
stream.append(&self.block_hash);
stream.out()
}
/// Try to restore manifest data from raw bytes, interpreted as RLP.
pub fn from_rlp(raw: &[u8]) -> Result<Self, DecoderError> {
let decoder = UntrustedRlp::new(raw);
let state_hashes: Vec<H256> = try!(decoder.val_at(0));
let block_hashes: Vec<H256> = try!(decoder.val_at(1));
let state_root: H256 = try!(decoder.val_at(2));
let block_number: u64 = try!(decoder.val_at(3));
let block_hash: H256 = try!(decoder.val_at(4));
Ok(ManifestData {
state_hashes: state_hashes,
block_hashes: block_hashes,
state_root: state_root,
block_number: block_number,
block_hash: block_hash,
})
}
}

2
ethcore/src/verification/mod.rs
View File

@ -25,7 +25,7 @@ pub use self::canon_verifier::CanonVerifier;
pub use self::noop_verifier::NoopVerifier;
/// Verifier type.
#[derive(Debug, PartialEq)]
#[derive(Debug, PartialEq, Clone)]
pub enum VerifierType {
/// Verifies block normally.
Canon,

3
parity/cli.rs
View File

@ -243,6 +243,8 @@ Snapshot Options:
index, hash, or 'latest'. Note that taking snapshots at
non-recent blocks will only work with --pruning archive
[default: latest]
--no-periodic-snapshot Disable automated snapshots which usually occur once
every 10000 blocks.
Virtual Machine Options:
--jitvm Enable the JIT VM.
@ -382,6 +384,7 @@ pub struct Args {
pub flag_from: String,
pub flag_to: String,
pub flag_at: String,
pub flag_no_periodic_snapshot: bool,
pub flag_format: Option<String>,
pub flag_jitvm: bool,
pub flag_log_file: Option<String>,

2
parity/configuration.rs
View File

@ -226,6 +226,7 @@ impl Configuration {
ui: self.args.cmd_ui,
name: self.args.flag_identity,
custom_bootnodes: self.args.flag_bootnodes.is_some(),
no_periodic_snapshot: self.args.flag_no_periodic_snapshot,
};
Cmd::Run(run_cmd)
};
@ -802,6 +803,7 @@ mod tests {
ui: false,
name: "".into(),
custom_bootnodes: false,
no_periodic_snapshot: false,
}));
}

11
parity/dapps.rs
View File

@ -18,6 +18,7 @@ use std::sync::Arc;
use io::PanicHandler;
use rpc_apis;
use ethcore::client::Client;
use ethsync::SyncProvider;
use helpers::replace_home;
#[derive(Debug, PartialEq, Clone)]
@ -49,6 +50,7 @@ pub struct Dependencies {
pub panic_handler: Arc<PanicHandler>,
pub apis: Arc<rpc_apis::Dependencies>,
pub client: Arc<Client>,
pub sync: Arc<SyncProvider>,
}
pub fn new(configuration: Configuration, deps: Dependencies) -> Result<Option<WebappServer>, String> {
@ -117,9 +119,12 @@ mod server {
) -> Result<WebappServer, String> {
use ethcore_dapps as dapps;
let server = dapps::ServerBuilder::new(dapps_path, Arc::new(Registrar {
client: deps.client.clone(),
}));
let mut server = dapps::ServerBuilder::new(
dapps_path,
Arc::new(Registrar { client: deps.client.clone() })
);
let sync = deps.sync.clone();
server.with_sync_status(Arc::new(move || sync.status().is_major_syncing()));
let server = rpc_apis::setup_rpc(server, deps.apis.clone(), rpc_apis::ApiSet::UnsafeContext);
let start_result = match auth {
None => {

22
parity/main.rs
View File

@ -99,9 +99,11 @@ mod modules;
mod account;
mod blockchain;
mod presale;
mod run;
mod sync;
mod snapshot;
mod run;
#[cfg(feature="ipc")]
mod sync;
#[cfg(feature="ipc")]
mod boot;
#[cfg(feature="stratum")]
@ -158,10 +160,24 @@ mod stratum_optional {
}
}
fn main() {
#[cfg(not(feature="ipc"))]
fn sync_main() -> bool {
false
}
#[cfg(feature="ipc")]
fn sync_main() -> bool {
// just redirect to the sync::main()
if std::env::args().nth(1).map_or(false, |arg| arg == "sync") {
sync::main();
true
} else {
false
}
}
fn main() {
if sync_main() {
return;
}

7
parity/modules.rs
View File

@ -18,6 +18,7 @@ use std::sync::Arc;
use ethcore::client::BlockChainClient;
use hypervisor::Hypervisor;
use ethsync::{SyncConfig, NetworkConfiguration, NetworkError};
use ethcore::snapshot::SnapshotService;
#[cfg(not(feature="ipc"))]
use self::no_ipc_deps::*;
#[cfg(feature="ipc")]
@ -25,10 +26,12 @@ use self::ipc_deps::*;
use ethcore_logger::Config as LogConfig;
use std::path::Path;
#[cfg(feature="ipc")]
pub mod service_urls {
use std::path::PathBuf;
pub const CLIENT: &'static str = "parity-chain.ipc";
pub const SNAPSHOT: &'static str = "parity-snapshot.ipc";
pub const SYNC: &'static str = "parity-sync.ipc";
pub const SYNC_NOTIFY: &'static str = "parity-sync-notify.ipc";
pub const NETWORK_MANAGER: &'static str = "parity-manage-net.ipc";
@ -119,6 +122,7 @@ pub fn sync
sync_cfg: SyncConfig,
net_cfg: NetworkConfiguration,
_client: Arc<BlockChainClient>,
_snapshot_service: Arc<SnapshotService>,
log_settings: &LogConfig,
)
-> Result<SyncModules, NetworkError>
@ -148,10 +152,11 @@ pub fn sync
sync_cfg: SyncConfig,
net_cfg: NetworkConfiguration,
client: Arc<BlockChainClient>,
snapshot_service: Arc<SnapshotService>,
_log_settings: &LogConfig,
)
-> Result<SyncModules, NetworkError>
{
let eth_sync = try!(EthSync::new(sync_cfg, client, net_cfg));
let eth_sync = try!(EthSync::new(sync_cfg, client, snapshot_service, net_cfg));
Ok((eth_sync.clone() as Arc<SyncProvider>, eth_sync.clone() as Arc<ManageNetwork>, eth_sync.clone() as Arc<ChainNotify>))
}

28
parity/run.rs
View File

@ -28,6 +28,7 @@ use ethcore::client::{Mode, Switch, DatabaseCompactionProfile, VMType, ChainNoti
use ethcore::service::ClientService;
use ethcore::account_provider::AccountProvider;
use ethcore::miner::{Miner, MinerService, ExternalMiner, MinerOptions};
use ethcore::snapshot;
use ethsync::SyncConfig;
use informant::Informant;
@ -46,6 +47,12 @@ use rpc_apis;
use rpc;
use url;
// how often to take periodic snapshots.
const SNAPSHOT_PERIOD: u64 = 10000;
// how many blocks to wait before starting a periodic snapshot.
const SNAPSHOT_HISTORY: u64 = 1000;
#[derive(Debug, PartialEq)]
pub struct RunCmd {
pub cache_config: CacheConfig,
@ -77,6 +84,7 @@ pub struct RunCmd {
pub ui: bool,
pub name: String,
pub custom_bootnodes: bool,
pub no_periodic_snapshot: bool,
}
pub fn execute(cmd: RunCmd) -> Result<(), String> {
@ -179,13 +187,14 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
// take handle to client
let client = service.client();
let snapshot_service = service.snapshot_service();
// create external miner
let external_miner = Arc::new(ExternalMiner::default());
// create sync object
let (sync_provider, manage_network, chain_notify) = try!(modules::sync(
&mut hypervisor, sync_config, net_conf.into(), client.clone(), &cmd.logger_config,
&mut hypervisor, sync_config, net_conf.into(), client.clone(), snapshot_service, &cmd.logger_config,
).map_err(|e| format!("Sync error: {}", e)));
service.add_notify(chain_notify.clone());
@ -224,6 +233,7 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
panic_handler: panic_handler.clone(),
apis: deps_for_rpc_apis.clone(),
client: client.clone(),
sync: sync_provider.clone(),
};
// start dapps server
@ -249,6 +259,22 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
});
service.register_io_handler(io_handler).expect("Error registering IO handler");
// the watcher must be kept alive.
let _watcher = match cmd.no_periodic_snapshot {
true => None,
false => {
let watcher = Arc::new(snapshot::Watcher::new(
service.client(),
service.io().channel(),
SNAPSHOT_PERIOD,
SNAPSHOT_HISTORY,
));
service.add_notify(watcher.clone());
Some(watcher)
},
};
// start ui
if cmd.ui {
if !cmd.dapps_conf.enabled {

7
parity/snapshot.rs
View File

@ -129,10 +129,9 @@ impl SnapshotCommand {
let informant_handle = snapshot.clone();
::std::thread::spawn(move || {
while let RestorationStatus::Ongoing = informant_handle.status() {
let (state_chunks, block_chunks) = informant_handle.chunks_done();
while let RestorationStatus::Ongoing { state_chunks_done, block_chunks_done } = informant_handle.status() {
info!("Processed {}/{} state chunks and {}/{} block chunks.",
state_chunks, num_state, block_chunks, num_blocks);
state_chunks_done, num_state, block_chunks_done, num_blocks);
::std::thread::sleep(Duration::from_secs(5));
}
@ -161,7 +160,7 @@ impl SnapshotCommand {
}
match snapshot.status() {
RestorationStatus::Ongoing => Err("Snapshot file is incomplete and missing chunks.".into()),
RestorationStatus::Ongoing { .. } => Err("Snapshot file is incomplete and missing chunks.".into()),
RestorationStatus::Failed => Err("Snapshot restoration failed.".into()),
RestorationStatus::Inactive => {
info!("Restoration complete.");

4
parity/sync.rs
View File

@ -20,6 +20,7 @@ use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use hypervisor::{SYNC_MODULE_ID, HYPERVISOR_IPC_URL, ControlService};
use ethcore::client::{RemoteClient, ChainNotify};
use ethcore::snapshot::{RemoteSnapshotService};
use ethsync::{SyncProvider, EthSync, ManageNetwork, ServiceConfiguration};
use modules::service_urls;
use boot;
@ -45,8 +46,9 @@ pub fn main() {
.unwrap_or_else(|e| panic!("Fatal: error reading boot arguments ({:?})", e));
let remote_client = dependency!(RemoteClient, &service_urls::with_base(&service_config.io_path, service_urls::CLIENT));
let remote_snapshot = dependency!(RemoteSnapshotService, &service_urls::with_base(&service_config.io_path, service_urls::SNAPSHOT));
let sync = EthSync::new(service_config.sync, remote_client.service().clone(), service_config.net).unwrap();
let sync = EthSync::new(service_config.sync, remote_client.service().clone(), remote_snapshot.service().clone(), service_config.net).unwrap();
let _ = boot::main_thread();
let service_stop = Arc::new(AtomicBool::new(false));

3
rpc/src/v1/impls/eth.rs
View File

@ -254,7 +254,8 @@ impl<C, S: ?Sized, M, EM> Eth for EthClient<C, S, M, EM> where
let status = take_weak!(self.sync).status();
let res = match status.state {
SyncState::Idle => SyncStatus::None,
SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead => {
SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead
| SyncState::SnapshotManifest | SyncState::SnapshotData | SyncState::SnapshotWaiting => {
let current_block = U256::from(take_weak!(self.client).chain_info().best_block_number);
let highest_block = U256::from(status.highest_block_number.unwrap_or(status.start_block_number));

2
rpc/src/v1/tests/helpers/sync_provider.rs
View File

@ -49,6 +49,8 @@ impl TestSyncProvider {
num_peers: config.num_peers,
num_active_peers: 0,
mem_used: 0,
num_snapshot_chunks: 0,
snapshot_chunks_done: 0,
}),
}
}

1
signer/Cargo.toml
View File

@ -19,6 +19,7 @@ ws = { git = "https://github.com/ethcore/ws-rs.git", branch = "mio-upstream-stab
ethcore-util = { path = "../util" }
ethcore-io = { path = "../util/io" }
ethcore-rpc = { path = "../rpc" }
ethcore-devtools = { path = "../devtools" }
parity-dapps-signer = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4", optional = true}
clippy = { version = "0.0.85", optional = true}

10
signer/src/lib.rs
View File

@ -54,15 +54,13 @@ extern crate jsonrpc_core;
extern crate ws;
#[cfg(feature = "ui")]
extern crate parity_dapps_signer as signer;
#[cfg(test)]
extern crate ethcore_devtools as devtools;
mod authcode_store;
mod ws_server;
#[cfg(test)]
mod tests;
pub use authcode_store::*;
pub use ws_server::*;
#[cfg(test)]
mod tests {
#[test]
fn should_work() {}
}

81
signer/src/tests/mod.rs Normal file
View File

@ -0,0 +1,81 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::env;
use std::thread;
use std::time::Duration;
use std::sync::Arc;
use devtools::http_client;
use rpc::ConfirmationsQueue;
use rand;
use ServerBuilder;
use Server;
pub fn serve() -> Server {
let queue = Arc::new(ConfirmationsQueue::default());
let builder = ServerBuilder::new(queue, env::temp_dir());
let port = 35000 + rand::random::<usize>() % 10000;
let res = builder.start(format!("127.0.0.1:{}", port).parse().unwrap()).unwrap();
thread::sleep(Duration::from_millis(25));
res
}
pub fn request(server: Server, request: &str) -> http_client::Response {
http_client::request(server.addr(), request)
}
#[test]
fn should_reject_invalid_host() {
// given
let server = serve();
// when
let response = request(server,
"\
GET / HTTP/1.1\r\n\
Host: test:8180\r\n\
Connection: close\r\n\
\r\n\
{}
"
);
// then
assert_eq!(response.status, "HTTP/1.1 403 FORBIDDEN".to_owned());
assert!(response.body.contains("URL Blocked"));
}
#[test]
fn should_serve_styles_even_on_disallowed_domain() {
// given
let server = serve();
// when
let response = request(server,
"\
GET /styles.css HTTP/1.1\r\n\
Host: test:8180\r\n\
Connection: close\r\n\
\r\n\
{}
"
);
// then
assert_eq!(response.status, "HTTP/1.1 200 OK".to_owned());
}

21
signer/src/ws_server/error_tpl.html Normal file
View File

@ -0,0 +1,21 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width">
{meta}
<title>{title}</title>
<link rel="stylesheet" href="/styles.css">
</head>
<body>
<div class="parity-navbar"></div>
<div class="parity-box">
<h1>{title}</h1>
<h3>{message}</h3>
<p><code>{details}</code></p>
</div>
<div class="parity-status">
<small>{version}</small>
</div>
</body>
</html>

9
signer/src/ws_server/mod.rs
View File

@ -93,9 +93,15 @@ pub struct Server {
broadcaster_handle: Option<thread::JoinHandle<()>>,
queue: Arc<ConfirmationsQueue>,
panic_handler: Arc<PanicHandler>,
addr: SocketAddr,
}
impl Server {
/// Returns the address this server is listening on
pub fn addr(&self) -> &SocketAddr {
&self.addr
}
/// Starts a new `WebSocket` server in separate thread.
/// Returns a `Server` handle which closes the server when droped.
fn start(addr: SocketAddr, handler: Arc<IoHandler>, queue: Arc<ConfirmationsQueue>, authcodes_path: PathBuf, skip_origin_validation: bool) -> Result<Server, ServerError> {
@ -121,7 +127,7 @@ impl Server {
// Spawn a thread with event loop
let handle = thread::spawn(move || {
ph.catch_panic(move || {
match ws.listen(addr).map_err(ServerError::from) {
match ws.listen(addr.clone()).map_err(ServerError::from) {
Err(ServerError::IoError(io)) => die(format!(
"Signer: Could not start listening on specified address. Make sure that no other instance is running on Signer's port. Details: {:?}",
io
@ -158,6 +164,7 @@ impl Server {
broadcaster_handle: Some(broadcaster_handle),
queue: queue,
panic_handler: panic_handler,
addr: addr,
})
}
}

46
signer/src/ws_server/session.rs
View File

@ -22,7 +22,7 @@ use std::path::{PathBuf, Path};
use std::sync::Arc;
use std::str::FromStr;
use jsonrpc_core::IoHandler;
use util::{H256, Mutex};
use util::{H256, Mutex, version};
#[cfg(feature = "ui")]
mod signer {
@ -107,21 +107,32 @@ impl ws::Handler for Session {
fn on_request(&mut self, req: &ws::Request) -> ws::Result<(ws::Response)> {
let origin = req.header("origin").or_else(|| req.header("Origin")).map(|x| &x[..]);
let host = req.header("host").or_else(|| req.header("Host")).map(|x| &x[..]);
// Styles file is allowed for error pages to display nicely.
let is_styles_file = req.resource() == "/styles.css";
// Check request origin and host header.
if !self.skip_origin_validation {
if !origin_is_allowed(&self.self_origin, origin) && !(origin.is_none() && origin_is_allowed(&self.self_origin, host)) {
let is_valid = origin_is_allowed(&self.self_origin, origin) || (origin.is_none() && origin_is_allowed(&self.self_origin, host));
let is_valid = is_styles_file || is_valid;
if !is_valid {
warn!(target: "signer", "Blocked connection to Signer API from untrusted origin.");
return Ok(ws::Response::forbidden(format!("You are not allowed to access system ui. Use: http://{}", self.self_origin)));
return Ok(error(
ErrorType::Forbidden,
"URL Blocked",
"You are not allowed to access Trusted Signer using this URL.",
Some(&format!("Use: http://{}", self.self_origin)),
));
}
}
// Detect if it's a websocket request.
if req.header("sec-websocket-key").is_some() {
// Detect if it's a websocket request
// (styles file skips origin validation, so make sure to prevent WS connections on this resource)
if req.header("sec-websocket-key").is_some() && !is_styles_file {
// Check authorization
if !auth_is_valid(&self.authcodes_path, req.protocols()) {
info!(target: "signer", "Unauthorized connection to Signer API blocked.");
return Ok(ws::Response::forbidden("You are not authorized.".into()));
return Ok(error(ErrorType::Forbidden, "Not Authorized", "Request to this API was not authorized.", None));
}
let protocols = req.protocols().expect("Existence checked by authorization.");
@ -137,7 +148,7 @@ impl ws::Handler for Session {
Ok(signer::handle(req.resource())
.map_or_else(
// return 404 not found
|| add_headers(ws::Response::not_found("Not found".into()), "text/plain"),
|| error(ErrorType::NotFound, "Not found", "Requested file was not found.", None),
// or serve the file
|f| add_headers(ws::Response::ok(f.content.into()), &f.mime)
))
@ -189,3 +200,24 @@ impl ws::Factory for Factory {
}
}
}
enum ErrorType {
NotFound,
Forbidden,
}
fn error(error: ErrorType, title: &str, message: &str, details: Option<&str>) -> ws::Response {
let content = format!(
include_str!("./error_tpl.html"),
title=title,
meta="",
message=message,
details=details.unwrap_or(""),
version=version(),
);
let res = match error {
ErrorType::NotFound => ws::Response::not_found(content),
ErrorType::Forbidden => ws::Response::forbidden(content),
};
add_headers(res, "text/html")
}

27
sync/src/api.rs
View File

@ -20,6 +20,7 @@ use network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId,
use util::{U256, H256};
use io::{TimerToken};
use ethcore::client::{BlockChainClient, ChainNotify};
use ethcore::snapshot::SnapshotService;
use ethcore::header::BlockNumber;
use sync_io::NetSyncIo;
use chain::{ChainSync, SyncStatus};
@ -71,12 +72,12 @@ pub struct EthSync {
impl EthSync {
/// Creates and register protocol with the network service
pub fn new(config: SyncConfig, chain: Arc<BlockChainClient>, network_config: NetworkConfiguration) -> Result<Arc<EthSync>, NetworkError> {
pub fn new(config: SyncConfig, chain: Arc<BlockChainClient>, snapshot_service: Arc<SnapshotService>, network_config: NetworkConfiguration) -> Result<Arc<EthSync>, NetworkError> {
let chain_sync = ChainSync::new(config, &*chain);
let service = try!(NetworkService::new(try!(network_config.into_basic())));
let sync = Arc::new(EthSync{
network: service,
handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain }),
handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain, snapshot_service: snapshot_service }),
});
Ok(sync)
@ -93,8 +94,10 @@ impl SyncProvider for EthSync {
}
struct SyncProtocolHandler {
/// Shared blockchain client. TODO: this should evetually become an IPC endpoint
/// Shared blockchain client.
chain: Arc<BlockChainClient>,
/// Shared snapshot service.
snapshot_service: Arc<SnapshotService>,
/// Sync strategy
sync: RwLock<ChainSync>,
}
@ -105,21 +108,21 @@ impl NetworkProtocolHandler for SyncProtocolHandler {
}
fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) {
ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain), *peer, packet_id, data);
ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer, packet_id, data);
}
fn connected(&self, io: &NetworkContext, peer: &PeerId) {
self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain), *peer);
self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer);
}
fn disconnected(&self, io: &NetworkContext, peer: &PeerId) {
self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain), *peer);
self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer);
}
fn timeout(&self, io: &NetworkContext, _timer: TimerToken) {
self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain));
self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain));
self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain));
self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
}
}
@ -133,7 +136,7 @@ impl ChainNotify for EthSync {
_duration: u64)
{
self.network.with_context(ETH_PROTOCOL, |context| {
let mut sync_io = NetSyncIo::new(context, &*self.handler.chain);
let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service);
self.handler.sync.write().chain_new_blocks(
&mut sync_io,
&imported,
@ -146,7 +149,7 @@ impl ChainNotify for EthSync {
fn start(&self) {
self.network.start().unwrap_or_else(|e| warn!("Error starting network: {:?}", e));
self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8])
self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8, 64u8])
.unwrap_or_else(|e| warn!("Error registering ethereum protocol: {:?}", e));
}
@ -202,7 +205,7 @@ impl ManageNetwork for EthSync {
fn stop_network(&self) {
self.network.with_context(ETH_PROTOCOL, |context| {
let mut sync_io = NetSyncIo::new(context, &*self.handler.chain);
let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service);
self.handler.sync.write().abort(&mut sync_io);
});
self.stop();

387
sync/src/chain.rs
View File

@ -96,17 +96,19 @@ use ethcore::header::{BlockNumber, Header as BlockHeader};
use ethcore::client::{BlockChainClient, BlockStatus, BlockID, BlockChainInfo, BlockImportError};
use ethcore::error::*;
use ethcore::block::Block;
use ethcore::snapshot::{ManifestData, RestorationStatus};
use sync_io::SyncIo;
use time;
use super::SyncConfig;
use blocks::BlockCollection;
use snapshot::{Snapshot, ChunkType};
use rand::{thread_rng, Rng};
known_heap_size!(0, PeerInfo);
type PacketDecodeError = DecoderError;
const PROTOCOL_VERSION: u8 = 63u8;
const PROTOCOL_VERSION: u8 = 64u8;
const MAX_BODIES_TO_SEND: usize = 256;
const MAX_HEADERS_TO_SEND: usize = 512;
const MAX_NODE_DATA_TO_SEND: usize = 1024;
@ -136,14 +138,26 @@ const GET_NODE_DATA_PACKET: u8 = 0x0d;
const NODE_DATA_PACKET: u8 = 0x0e;
const GET_RECEIPTS_PACKET: u8 = 0x0f;
const RECEIPTS_PACKET: u8 = 0x10;
const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11;
const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12;
const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13;
const SNAPSHOT_DATA_PACKET: u8 = 0x14;
const HEADERS_TIMEOUT_SEC: f64 = 15f64;
const BODIES_TIMEOUT_SEC: f64 = 5f64;
const FORK_HEADER_TIMEOUT_SEC: f64 = 3f64;
const SNAPSHOT_MANIFEST_TIMEOUT_SEC: f64 = 3f64;
const SNAPSHOT_DATA_TIMEOUT_SEC: f64 = 10f64;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
/// Sync state
pub enum SyncState {
/// Waiting for pv64 peers to start snapshot syncing
SnapshotManifest,
/// Downloading snapshot data
SnapshotData,
/// Waiting for snapshot restoration to complete
SnapshotWaiting,
/// Downloading subchain heads
ChainHead,
/// Initial chain sync complete. Waiting for new packets
@ -177,10 +191,14 @@ pub struct SyncStatus {
pub blocks_received: BlockNumber,
/// Total number of connected peers
pub num_peers: usize,
/// Total number of active peers
/// Total number of active peers.
pub num_active_peers: usize,
/// Heap memory used in bytes
/// Heap memory used in bytes.
pub mem_used: usize,
/// Snapshot chunks
pub num_snapshot_chunks: usize,
/// Snapshot chunks downloaded
pub snapshot_chunks_done: usize,
}
impl SyncStatus {
@ -207,6 +225,8 @@ enum PeerAsking {
BlockHeaders,
BlockBodies,
Heads,
SnapshotManifest,
SnapshotData,
}
#[derive(Clone, Eq, PartialEq)]
@ -240,6 +260,8 @@ struct PeerInfo {
asking_blocks: Vec<H256>,
/// Holds requested header hash if currently requesting block header by hash
asking_hash: Option<H256>,
/// Holds requested snapshot chunk hash if any.
asking_snapshot_data: Option<H256>,
/// Request timestamp
ask_time: f64,
/// Holds a set of transactions recently sent to this peer to avoid spamming.
@ -248,6 +270,10 @@ struct PeerInfo {
expired: bool,
/// Peer fork confirmation status
confirmation: ForkConfirmation,
/// Best snapshot hash
snapshot_hash: Option<H256>,
/// Best snapshot block number
snapshot_number: Option<BlockNumber>,
}
impl PeerInfo {
@ -293,6 +319,8 @@ pub struct ChainSync {
network_id: U256,
/// Optional fork block to check
fork_block: Option<(BlockNumber, H256)>,
/// Snapshot downloader.
snapshot: Snapshot,
}
type RlpResponseResult = Result<Option<(PacketId, RlpStream)>, PacketDecodeError>;
@ -301,8 +329,8 @@ impl ChainSync {
/// Create a new instance of syncing strategy.
pub fn new(config: SyncConfig, chain: &BlockChainClient) -> ChainSync {
let chain = chain.chain_info();
let mut sync = ChainSync {
state: SyncState::ChainHead,
ChainSync {
state: SyncState::Idle,
starting_block: chain.best_block_number,
highest_block: None,
last_imported_block: chain.best_block_number,
@ -317,16 +345,15 @@ impl ChainSync {
_max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks),
network_id: config.network_id,
fork_block: config.fork_block,
};
sync.reset();
sync
snapshot: Snapshot::new(),
}
}
/// @returns Synchonization status
pub fn status(&self) -> SyncStatus {
SyncStatus {
state: self.state.clone(),
protocol_version: 63,
protocol_version: if self.state == SyncState::SnapshotData { 64 } else { 63 },
network_id: self.network_id,
start_block_number: self.starting_block,
last_imported_block_number: Some(self.last_imported_block),
@ -335,6 +362,8 @@ impl ChainSync {
blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 },
num_peers: self.peers.values().filter(|p| p.is_allowed()).count(),
num_active_peers: self.peers.values().filter(|p| p.is_allowed() && p.asking != PeerAsking::Nothing).count(),
num_snapshot_chunks: self.snapshot.total_chunks(),
snapshot_chunks_done: self.snapshot.done_chunks(),
mem_used:
self.blocks.heap_size()
+ self.peers.heap_size_of_children()
@ -350,8 +379,13 @@ impl ChainSync {
#[cfg_attr(feature="dev", allow(for_kv_map))] // Because it's not possible to get `values_mut()`
/// Reset sync. Clear all downloaded data but keep the queue
fn reset(&mut self) {
fn reset(&mut self, io: &mut SyncIo) {
self.blocks.clear();
self.snapshot.clear();
if self.state == SyncState::SnapshotData {
debug!(target:"sync", "Aborting snapshot restore");
io.snapshot_service().abort_restore();
}
for (_, ref mut p) in &mut self.peers {
p.asking_blocks.clear();
p.asking_hash = None;
@ -368,7 +402,7 @@ impl ChainSync {
/// Restart sync
pub fn restart(&mut self, io: &mut SyncIo) {
trace!(target: "sync", "Restarting");
self.reset();
self.reset(io);
self.start_sync_round(io);
self.continue_sync(io);
}
@ -380,13 +414,19 @@ impl ChainSync {
if self.active_peers.is_empty() {
trace!(target: "sync", "No more active peers");
if self.state == SyncState::ChainHead {
self.complete_sync();
self.complete_sync(io);
} else {
self.restart(io);
}
}
}
fn start_snapshot_sync(&mut self, io: &mut SyncIo, peer_id: PeerId) {
self.snapshot.clear();
self.request_snapshot_manifest(io, peer_id);
self.state = SyncState::SnapshotManifest;
}
/// Restart sync after bad block has been detected. May end up re-downloading up to QUEUE_SIZE blocks
fn restart_on_bad_block(&mut self, io: &mut SyncIo) {
// Do not assume that the block queue/chain still has our last_imported_block
@ -398,8 +438,9 @@ impl ChainSync {
/// Called by peer to report status
fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
let protocol_version: u32 = try!(r.val_at(0));
let peer = PeerInfo {
protocol_version: try!(r.val_at(0)),
protocol_version: protocol_version,
network_id: try!(r.val_at(1)),
difficulty: Some(try!(r.val_at(2))),
latest_hash: try!(r.val_at(3)),
@ -412,6 +453,9 @@ impl ChainSync {
last_sent_transactions: HashSet::new(),
expired: false,
confirmation: if self.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed },
asking_snapshot_data: None,
snapshot_hash: if protocol_version == 64 { Some(try!(r.val_at(5))) } else { None },
snapshot_number: if protocol_version == 64 { Some(try!(r.val_at(6))) } else { None },
};
trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis);
@ -749,6 +793,96 @@ impl ChainSync {
Ok(())
}
/// Called when snapshot manifest is downloaded from a peer.
fn on_snapshot_manifest(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id);
return Ok(());
}
self.clear_peer_download(peer_id);
if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || self.state != SyncState::SnapshotManifest {
trace!(target: "sync", "{}: Ignored unexpected manifest", peer_id);
self.continue_sync(io);
return Ok(());
}
let manifest_rlp = try!(r.at(0));
let manifest = match ManifestData::from_rlp(&manifest_rlp.as_raw()) {
Err(e) => {
trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e);
io.disconnect_peer(peer_id);
self.continue_sync(io);
return Ok(());
}
Ok(manifest) => manifest,
};
self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3());
io.snapshot_service().begin_restore(manifest);
self.state = SyncState::SnapshotData;
// give a task to the same peer first.
self.sync_peer(io, peer_id, false);
// give tasks to other peers
self.continue_sync(io);
Ok(())
}
/// Called when snapshot data is downloaded from a peer.
fn on_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id);
return Ok(());
}
self.clear_peer_download(peer_id);
if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || self.state != SyncState::SnapshotData {
trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id);
self.continue_sync(io);
return Ok(());
}
// check service status
match io.snapshot_service().status() {
RestorationStatus::Inactive | RestorationStatus::Failed => {
trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id);
self.state = SyncState::Idle;
self.snapshot.clear();
self.continue_sync(io);
return Ok(());
},
RestorationStatus::Ongoing { .. } => {
trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id);
},
}
let snapshot_data: Bytes = try!(r.val_at(0));
match self.snapshot.validate_chunk(&snapshot_data) {
Ok(ChunkType::Block(hash)) => {
trace!(target: "sync", "{}: Processing block chunk", peer_id);
io.snapshot_service().restore_block_chunk(hash, snapshot_data);
}
Ok(ChunkType::State(hash)) => {
trace!(target: "sync", "{}: Processing state chunk", peer_id);
io.snapshot_service().restore_state_chunk(hash, snapshot_data);
}
Err(()) => {
trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id);
io.disconnect_peer(peer_id);
self.continue_sync(io);
return Ok(());
}
}
if self.snapshot.is_complete() {
// wait for snapshot restoration process to complete
self.state = SyncState::SnapshotWaiting;
}
// give a task to the same peer first.
self.sync_peer(io, peer_id, false);
// give tasks to other peers
self.continue_sync(io);
Ok(())
}
/// Called by peer when it is disconnecting
pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) {
trace!(target: "sync", "== Disconnecting {}: {}", peer, io.peer_info(peer));
@ -764,7 +898,7 @@ impl ChainSync {
/// Called when a new peer is connected
pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) {
trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer));
if let Err(e) = self.send_status(io) {
if let Err(e) = self.send_status(io, peer) {
debug!(target:"sync", "Error sending status request: {:?}", e);
io.disable_peer(peer);
}
@ -772,24 +906,27 @@ impl ChainSync {
/// Resume downloading
fn continue_sync(&mut self, io: &mut SyncIo) {
let mut peers: Vec<(PeerId, U256)> = self.peers.iter().filter_map(|(k, p)|
if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero))) } else { None }).collect();
let mut peers: Vec<(PeerId, U256, u32)> = self.peers.iter().filter_map(|(k, p)|
if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero), p.protocol_version)) } else { None }).collect();
thread_rng().shuffle(&mut peers); //TODO: sort by rating
// prefer peers with higher protocol version
peers.sort_by(|&(_, _, ref v1), &(_, _, ref v2)| v1.cmp(v2));
trace!(target: "sync", "Syncing with {}/{} peers", self.active_peers.len(), peers.len());
for (p, _) in peers {
for (p, _, _) in peers {
if self.active_peers.contains(&p) {
self.sync_peer(io, p, false);
}
}
if self.state != SyncState::Waiting && !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) {
self.complete_sync();
if self.state != SyncState::Waiting && self.state != SyncState::SnapshotWaiting
&& !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) {
self.complete_sync(io);
}
}
/// Called after all blocks have been downloaded
fn complete_sync(&mut self) {
fn complete_sync(&mut self, io: &mut SyncIo) {
trace!(target: "sync", "Sync complete");
self.reset();
self.reset(io);
self.state = SyncState::Idle;
}
@ -805,7 +942,7 @@ impl ChainSync {
trace!(target: "sync", "Skipping deactivated peer");
return;
}
let (peer_latest, peer_difficulty) = {
let (peer_latest, peer_difficulty, peer_snapshot_number, peer_snapshot_hash) = {
let peer = self.peers.get_mut(&peer_id).unwrap();
if peer.asking != PeerAsking::Nothing || !peer.can_sync() {
return;
@ -814,7 +951,11 @@ impl ChainSync {
trace!(target: "sync", "Waiting for the block queue");
return;
}
(peer.latest_hash.clone(), peer.difficulty.clone())
if self.state == SyncState::SnapshotWaiting {
trace!(target: "sync", "Waiting for the snapshot restoration");
return;
}
(peer.latest_hash.clone(), peer.difficulty.clone(), peer.snapshot_number.as_ref().cloned(), peer.snapshot_hash.as_ref().cloned())
};
let chain_info = io.chain().chain_info();
let td = chain_info.pending_total_difficulty;
@ -823,13 +964,18 @@ impl ChainSync {
if force || self.state == SyncState::NewBlocks || peer_difficulty.map_or(true, |pd| pd > syncing_difficulty) {
match self.state {
SyncState::Idle => {
if self.last_imported_block < chain_info.best_block_number {
self.last_imported_block = chain_info.best_block_number;
self.last_imported_hash = chain_info.best_block_hash;
// check if we can start snapshot sync with this peer
if peer_snapshot_number.unwrap_or(0) > 0 && chain_info.best_block_number == 0 {
self.start_snapshot_sync(io, peer_id);
} else {
if self.last_imported_block < chain_info.best_block_number {
self.last_imported_block = chain_info.best_block_number;
self.last_imported_hash = chain_info.best_block_hash;
}
trace!(target: "sync", "Starting sync with {}", peer_id);
self.start_sync_round(io);
self.sync_peer(io, peer_id, force);
}
trace!(target: "sync", "Starting sync with {}", peer_id);
self.start_sync_round(io);
self.sync_peer(io, peer_id, force);
},
SyncState::ChainHead => {
// Request subchain headers
@ -843,8 +989,14 @@ impl ChainSync {
if io.chain().block_status(BlockID::Hash(peer_latest)) == BlockStatus::Unknown {
self.request_blocks(io, peer_id, false);
}
}
SyncState::Waiting => ()
},
SyncState::SnapshotData => {
if peer_snapshot_hash.is_some() && peer_snapshot_hash == self.snapshot.snapshot_hash() {
self.request_snapshot_data(io, peer_id);
}
},
SyncState::SnapshotManifest => (), //already downloading from other peer
SyncState::Waiting | SyncState::SnapshotWaiting => ()
}
}
}
@ -903,6 +1055,16 @@ impl ChainSync {
}
}
/// Find some headers or blocks to download for a peer.
fn request_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId) {
self.clear_peer_download(peer_id);
// find chunk data to download
if let Some(hash) = self.snapshot.needed_chunk() {
self.peers.get_mut(&peer_id).unwrap().asking_snapshot_data = Some(hash.clone());
self.request_snapshot_chunk(io, peer_id, &hash);
}
}
/// Clear all blocks/headers marked as being downloaded by a peer.
fn clear_peer_download(&mut self, peer_id: PeerId) {
let peer = self.peers.get_mut(&peer_id).unwrap();
@ -917,9 +1079,15 @@ impl ChainSync {
self.blocks.clear_body_download(b);
}
},
PeerAsking::SnapshotData => {
if let Some(hash) = peer.asking_snapshot_data {
self.snapshot.clear_chunk_download(&hash);
}
},
_ => (),
}
peer.asking_blocks.clear();
peer.asking_snapshot_data = None;
}
fn block_imported(&mut self, hash: &H256, number: BlockNumber, parent: &H256) {
@ -1016,6 +1184,22 @@ impl ChainSync {
rlp.append(&if reverse {1u32} else {0u32});
self.send_request(sync, peer_id, asking, GET_BLOCK_HEADERS_PACKET, rlp.out());
}
/// Request snapshot manifest from a peer.
fn request_snapshot_manifest(&mut self, sync: &mut SyncIo, peer_id: PeerId) {
trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id);
let rlp = RlpStream::new_list(0);
self.send_request(sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out());
}
/// Request snapshot chunk from a peer.
fn request_snapshot_chunk(&mut self, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) {
trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk);
let mut rlp = RlpStream::new_list(1);
rlp.append(chunk);
self.send_request(sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out());
}
/// Request block bodies from a peer
fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec<H256>) {
let mut rlp = RlpStream::new_list(hashes.len());
@ -1086,14 +1270,22 @@ impl ChainSync {
}
/// Send Status message
fn send_status(&mut self, io: &mut SyncIo) -> Result<(), NetworkError> {
let mut packet = RlpStream::new_list(5);
fn send_status(&mut self, io: &mut SyncIo, peer: PeerId) -> Result<(), NetworkError> {
let pv64 = io.eth_protocol_version(peer) >= 64;
let mut packet = RlpStream::new_list(if pv64 { 7 } else { 5 });
let chain = io.chain().chain_info();
packet.append(&(PROTOCOL_VERSION as u32));
packet.append(&self.network_id);
packet.append(&chain.total_difficulty);
packet.append(&chain.best_block_hash);
packet.append(&chain.genesis_hash);
if pv64 {
let manifest = io.snapshot_service().manifest();
let block_number = manifest.as_ref().map_or(0, |m| m.block_number);
let manifest_hash = manifest.map_or(H256::new(), |m| m.into_rlp().sha3());
packet.append(&manifest_hash);
packet.append(&block_number);
}
io.respond(STATUS_PACKET, packet.out())
}
@ -1230,6 +1422,48 @@ impl ChainSync {
Ok(Some((RECEIPTS_PACKET, rlp_result)))
}
/// Respond to GetSnapshotManifest request
fn return_snapshot_manifest(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult {
let count = r.item_count();
trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id);
if count != 0 {
debug!(target: "sync", "Invalid GetSnapshotManifest request, ignoring.");
return Ok(None);
}
let rlp = match io.snapshot_service().manifest() {
Some(manifest) => {
trace!(target: "sync", "{} <- SnapshotManifest", peer_id);
let mut rlp = RlpStream::new_list(1);
rlp.append_raw(&manifest.into_rlp(), 1);
rlp
},
None => {
trace!(target: "sync", "{}: No manifest to return", peer_id);
let rlp = RlpStream::new_list(0);
rlp
}
};
Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp)))
}
/// Respond to GetSnapshotManifest request
fn return_snapshot_data(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult {
let hash: H256 = try!(r.val_at(0));
trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash);
let rlp = match io.snapshot_service().chunk(hash) {
Some(data) => {
let mut rlp = RlpStream::new_list(1);
rlp.append(&data);
rlp
},
None => {
let rlp = RlpStream::new_list(0);
rlp
}
};
Ok(Some((SNAPSHOT_DATA_PACKET, rlp)))
}
fn return_rlp<FRlp, FError>(io: &mut SyncIo, rlp: &UntrustedRlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError>
where FRlp : Fn(&SyncIo, &UntrustedRlp, PeerId) -> RlpResponseResult,
FError : FnOnce(NetworkError) -> String
@ -1266,6 +1500,14 @@ impl ChainSync {
ChainSync::return_node_data,
|e| format!("Error sending nodes: {:?}", e)),
GET_SNAPSHOT_MANIFEST_PACKET => ChainSync::return_rlp(io, &rlp, peer,
ChainSync::return_snapshot_manifest,
|e| format!("Error sending snapshot manifest: {:?}", e)),
GET_SNAPSHOT_DATA_PACKET => ChainSync::return_rlp(io, &rlp, peer,
ChainSync::return_snapshot_data,
|e| format!("Error sending snapshot data: {:?}", e)),
_ => {
sync.write().on_packet(io, peer, packet_id, data);
Ok(())
@ -1289,6 +1531,8 @@ impl ChainSync {
BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp),
NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp),
NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp),
SNAPSHOT_MANIFEST_PACKET => self.on_snapshot_manifest(io, peer, &rlp),
SNAPSHOT_DATA_PACKET => self.on_snapshot_data(io, peer, &rlp),
_ => {
debug!(target: "sync", "Unknown packet {}", packet_id);
Ok(())
@ -1308,6 +1552,8 @@ impl ChainSync {
PeerAsking::BlockBodies => (tick - peer.ask_time) > BODIES_TIMEOUT_SEC,
PeerAsking::Nothing => false,
PeerAsking::ForkHeader => (tick - peer.ask_time) > FORK_HEADER_TIMEOUT_SEC,
PeerAsking::SnapshotManifest => (tick - peer.ask_time) > SNAPSHOT_MANIFEST_TIMEOUT_SEC,
PeerAsking::SnapshotData => (tick - peer.ask_time) > SNAPSHOT_DATA_TIMEOUT_SEC,
};
if timeout {
trace!(target:"sync", "Timeout {}", peer_id);
@ -1321,9 +1567,12 @@ impl ChainSync {
}
fn check_resume(&mut self, io: &mut SyncIo) {
if !io.chain().queue_info().is_full() && self.state == SyncState::Waiting {
if self.state == SyncState::Waiting && !io.chain().queue_info().is_full() && self.state == SyncState::Waiting {
self.state = SyncState::Blocks;
self.continue_sync(io);
} else if self.state == SyncState::SnapshotWaiting && io.snapshot_service().status() == RestorationStatus::Inactive {
self.state = SyncState::Idle;
self.continue_sync(io);
}
}
@ -1559,6 +1808,7 @@ impl ChainSync {
#[cfg(test)]
mod tests {
use tests::helpers::*;
use tests::snapshot::TestSnapshotService;
use super::*;
use ::SyncConfig;
use util::*;
@ -1612,7 +1862,8 @@ mod tests {
fn return_receipts_empty() {
let mut client = TestBlockChainClient::new();
let mut queue = VecDeque::new();
let io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let result = ChainSync::return_receipts(&io, &UntrustedRlp::new(&[0xc0]), 0);
@ -1624,7 +1875,8 @@ mod tests {
let mut client = TestBlockChainClient::new();
let mut queue = VecDeque::new();
let sync = dummy_sync_with_peer(H256::new(), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let mut receipt_list = RlpStream::new_list(4);
receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
@ -1679,7 +1931,8 @@ mod tests {
let hashes: Vec<_> = headers.iter().map(|h| HeaderView::new(h).sha3()).collect();
let mut queue = VecDeque::new();
let io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let unknown: H256 = H256::new();
let result = ChainSync::return_block_headers(&io, &UntrustedRlp::new(&make_hash_req(&unknown, 1, 0, false)), 0);
@ -1717,7 +1970,8 @@ mod tests {
let mut client = TestBlockChainClient::new();
let mut queue = VecDeque::new();
let sync = dummy_sync_with_peer(H256::new(), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let mut node_list = RlpStream::new_list(3);
node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
@ -1758,6 +2012,9 @@ mod tests {
last_sent_transactions: HashSet::new(),
expired: false,
confirmation: super::ForkConfirmation::Confirmed,
snapshot_number: None,
snapshot_hash: None,
asking_snapshot_data: None,
});
sync
}
@ -1769,7 +2026,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client);
let chain_info = client.chain_info();
let io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let lagging_peers = sync.get_lagging_peers(&chain_info, &io);
@ -1800,7 +2058,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let chain_info = client.chain_info();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peers = sync.get_lagging_peers(&chain_info, &io);
let peer_count = sync.propagate_new_hashes(&chain_info, &mut io, &peers);
@ -1820,7 +2079,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let chain_info = client.chain_info();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peers = sync.get_lagging_peers(&chain_info, &io);
let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[], &peers);
@ -1840,7 +2100,8 @@ mod tests {
let hash = client.block_hash(BlockID::Number(99)).unwrap();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let chain_info = client.chain_info();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peers = sync.get_lagging_peers(&chain_info, &io);
let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[hash.clone()], &peers);
@ -1859,7 +2120,8 @@ mod tests {
client.insert_transaction_to_queue();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
let mut queue = VecDeque::new();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peer_count = sync.propagate_new_transactions(&mut io);
// Try to propagate same transactions for the second time
let peer_count2 = sync.propagate_new_transactions(&mut io);
@ -1880,7 +2142,8 @@ mod tests {
client.insert_transaction_to_queue();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
let mut queue = VecDeque::new();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peer_count = sync.propagate_new_transactions(&mut io);
sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[]);
// Try to propagate same transactions for the second time
@ -1903,17 +2166,17 @@ mod tests {
client.insert_transaction_to_queue();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
let mut queue = VecDeque::new();
let mut ss = TestSnapshotService::new();
// should sent some
{
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peer_count = sync.propagate_new_transactions(&mut io);
assert_eq!(1, io.queue.len());
assert_eq!(1, peer_count);
}
// Insert some more
client.insert_transaction_to_queue();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
// Propagate new transactions
let peer_count2 = sync.propagate_new_transactions(&mut io);
// And now the peer should have all transactions
@ -1939,7 +2202,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
//sync.have_common_block = true;
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let block = UntrustedRlp::new(&block_data);
@ -1957,7 +2221,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let block = UntrustedRlp::new(&block_data);
@ -1972,7 +2237,8 @@ mod tests {
client.add_blocks(10, EachBlockWith::Uncle);
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let empty_data = vec![];
let block = UntrustedRlp::new(&empty_data);
@ -1988,7 +2254,8 @@ mod tests {
client.add_blocks(10, EachBlockWith::Uncle);
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let hashes_data = get_dummy_hashes();
let hashes_rlp = UntrustedRlp::new(&hashes_data);
@ -2004,7 +2271,8 @@ mod tests {
client.add_blocks(10, EachBlockWith::Uncle);
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let empty_hashes_data = vec![];
let hashes_rlp = UntrustedRlp::new(&empty_hashes_data);
@ -2023,7 +2291,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let chain_info = client.chain_info();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peers = sync.get_lagging_peers(&chain_info, &io);
sync.propagate_new_hashes(&chain_info, &mut io, &peers);
@ -2042,7 +2311,8 @@ mod tests {
let mut queue = VecDeque::new();
let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
let chain_info = client.chain_info();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
let peers = sync.get_lagging_peers(&chain_info, &io);
sync.propagate_blocks(&chain_info, &mut io, &[], &peers);
@ -2076,7 +2346,8 @@ mod tests {
// when
{
let mut queue = VecDeque::new();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks);
sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]);
assert_eq!(io.chain.miner.status().transactions_in_future_queue, 0);
@ -2090,7 +2361,8 @@ mod tests {
}
{
let mut queue = VecDeque::new();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
io.chain.miner.chain_new_blocks(io.chain, &[], &[], &good_blocks, &retracted_blocks);
sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks, &[]);
}
@ -2114,7 +2386,8 @@ mod tests {
let retracted_blocks = vec![client.block_hash_delta_minus(1)];
let mut queue = VecDeque::new();
let mut io = TestIo::new(&mut client, &mut queue, None);
let mut ss = TestSnapshotService::new();
let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
// when
sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]);

36
sync/src/lib.rs
View File

@ -26,40 +26,6 @@
//! Implements ethereum protocol version 63 as specified here:
//! https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol
//!
//! Usage example:
//!
//! ```rust
//! extern crate ethcore_util as util;
//! extern crate ethcore_io as io;
//! extern crate ethcore;
//! extern crate ethsync;
//! use std::env;
//! use io::IoChannel;
//! use ethcore::client::{Client, ClientConfig};
//! use ethsync::{EthSync, SyncConfig, ManageNetwork, NetworkConfiguration};
//! use ethcore::ethereum;
//! use ethcore::miner::{GasPricer, Miner};
//!
//! fn main() {
//! let dir = env::temp_dir();
//! let spec = ethereum::new_frontier();
//! let miner = Miner::new(
//! Default::default(),
//! GasPricer::new_fixed(20_000_000_000u64.into()),
//! &spec,
//! None
//! );
//! let client = Client::new(
//! ClientConfig::default(),
//! &spec,
//! &dir,
//! miner,
//! IoChannel::disconnected()
//! ).unwrap();
//! let sync = EthSync::new(SyncConfig::default(), client, NetworkConfiguration::from(NetworkConfiguration::new())).unwrap();
//! sync.start_network();
//! }
//! ```
extern crate ethcore_network as network;
extern crate ethcore_io as io;
@ -83,6 +49,7 @@ extern crate ethcore_ipc as ipc;
mod chain;
mod blocks;
mod sync_io;
mod snapshot;
#[cfg(test)]
mod tests;
@ -96,4 +63,3 @@ pub use api::{EthSync, SyncProvider, SyncClient, NetworkManagerClient, ManageNet
ServiceConfiguration, NetworkConfiguration};
pub use chain::{SyncStatus, SyncState};
pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError};

200
sync/src/snapshot.rs Normal file
View File

@ -0,0 +1,200 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use util::{H256, Hashable};
use std::collections::HashSet;
use ethcore::snapshot::ManifestData;
#[derive(PartialEq, Eq, Debug)]
pub enum ChunkType {
State(H256),
Block(H256),
}
pub struct Snapshot {
pending_state_chunks: Vec<H256>,
pending_block_chunks: Vec<H256>,
downloading_chunks: HashSet<H256>,
completed_chunks: HashSet<H256>,
snapshot_hash: Option<H256>,
}
impl Snapshot {
/// Create a new instance.
pub fn new() -> Snapshot {
Snapshot {
pending_state_chunks: Vec::new(),
pending_block_chunks: Vec::new(),
downloading_chunks: HashSet::new(),
completed_chunks: HashSet::new(),
snapshot_hash: None,
}
}
/// Clear everything.
pub fn clear(&mut self) {
self.pending_state_chunks.clear();
self.pending_block_chunks.clear();
self.downloading_chunks.clear();
self.completed_chunks.clear();
self.snapshot_hash = None;
}
/// Reset collection for a manifest RLP
pub fn reset_to(&mut self, manifest: &ManifestData, hash: &H256) {
self.clear();
self.pending_state_chunks = manifest.state_hashes.clone();
self.pending_block_chunks = manifest.block_hashes.clone();
self.snapshot_hash = Some(hash.clone());
}
/// Validate chunk and mark it as downloaded
pub fn validate_chunk(&mut self, chunk: &[u8]) -> Result<ChunkType, ()> {
let hash = chunk.sha3();
if self.completed_chunks.contains(&hash) {
trace!(target: "sync", "Ignored proccessed chunk: {}", hash.hex());
return Err(());
}
self.downloading_chunks.remove(&hash);
if self.pending_block_chunks.iter().any(|h| h == &hash) {
self.completed_chunks.insert(hash.clone());
return Ok(ChunkType::Block(hash));
}
if self.pending_state_chunks.iter().any(|h| h == &hash) {
self.completed_chunks.insert(hash.clone());
return Ok(ChunkType::State(hash));
}
trace!(target: "sync", "Ignored unknown chunk: {}", hash.hex());
Err(())
}
/// Find a chunk to download
pub fn needed_chunk(&mut self) -> Option<H256> {
// check state chunks first
let mut chunk = self.pending_state_chunks.iter()
.find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h))
.cloned();
if chunk.is_none() {
chunk = self.pending_block_chunks.iter()
.find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h))
.cloned();
}
if let Some(hash) = chunk {
self.downloading_chunks.insert(hash.clone());
}
chunk
}
pub fn clear_chunk_download(&mut self, hash: &H256) {
self.downloading_chunks.remove(hash);
}
pub fn snapshot_hash(&self) -> Option<H256> {
self.snapshot_hash
}
pub fn total_chunks(&self) -> usize {
self.pending_block_chunks.len() + self.pending_state_chunks.len()
}
pub fn done_chunks(&self) -> usize {
self.total_chunks() - self.completed_chunks.len()
}
pub fn is_complete(&self) -> bool {
self.total_chunks() == self.completed_chunks.len()
}
}
#[cfg(test)]
mod test {
use util::*;
use super::*;
use ethcore::snapshot::ManifestData;
fn is_empty(snapshot: &Snapshot) -> bool {
snapshot.pending_block_chunks.is_empty() &&
snapshot.pending_state_chunks.is_empty() &&
snapshot.completed_chunks.is_empty() &&
snapshot.downloading_chunks.is_empty() &&
snapshot.snapshot_hash.is_none()
}
fn test_manifest() -> (ManifestData, H256, Vec<Bytes>, Vec<Bytes>) {
let state_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
let block_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
let manifest = ManifestData {
state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
state_root: H256::new(),
block_number: 42,
block_hash: H256::new(),
};
let mhash = manifest.clone().into_rlp().sha3();
(manifest, mhash, state_chunks, block_chunks)
}
#[test]
fn create_clear() {
let mut snapshot = Snapshot::new();
assert!(is_empty(&snapshot));
let (manifest, mhash, _, _,) = test_manifest();
snapshot.reset_to(&manifest, &mhash);
assert!(!is_empty(&snapshot));
snapshot.clear();
assert!(is_empty(&snapshot));
}
#[test]
fn validate_chunks() {
let mut snapshot = Snapshot::new();
let (manifest, mhash, state_chunks, block_chunks) = test_manifest();
snapshot.reset_to(&manifest, &mhash);
assert!(snapshot.validate_chunk(&H256::random().to_vec()).is_err());
let requested: Vec<H256> = (0..40).map(|_| snapshot.needed_chunk().unwrap()).collect();
assert!(snapshot.needed_chunk().is_none());
assert_eq!(&requested[0..20], &manifest.state_hashes[..]);
assert_eq!(&requested[20..40], &manifest.block_hashes[..]);
assert_eq!(snapshot.downloading_chunks.len(), 40);
assert_eq!(snapshot.validate_chunk(&state_chunks[4]), Ok(ChunkType::State(manifest.state_hashes[4].clone())));
assert_eq!(snapshot.completed_chunks.len(), 1);
assert_eq!(snapshot.downloading_chunks.len(), 39);
assert_eq!(snapshot.validate_chunk(&block_chunks[10]), Ok(ChunkType::Block(manifest.block_hashes[10].clone())));
assert_eq!(snapshot.completed_chunks.len(), 2);
assert_eq!(snapshot.downloading_chunks.len(), 38);
for (i, data) in state_chunks.iter().enumerate() {
if i != 4 {
assert!(snapshot.validate_chunk(data).is_ok());
}
}
for (i, data) in block_chunks.iter().enumerate() {
if i != 10 {
assert!(snapshot.validate_chunk(data).is_ok());
}
}
assert!(snapshot.is_complete());
assert_eq!(snapshot.snapshot_hash(), Some(manifest.into_rlp().sha3()));
}
}

20
sync/src/sync_io.rs
View File

@ -16,6 +16,8 @@
use network::{NetworkContext, PeerId, PacketId, NetworkError};
use ethcore::client::BlockChainClient;
use ethcore::snapshot::SnapshotService;
use api::ETH_PROTOCOL;
/// IO interface for the syning handler.
/// Provides peer connection management and an interface to the blockchain client.
@ -31,10 +33,14 @@ pub trait SyncIo {
fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec<u8>) -> Result<(), NetworkError>;
/// Get the blockchain
fn chain(&self) -> &BlockChainClient;
/// Get the snapshot service.
fn snapshot_service(&self) -> &SnapshotService;
/// Returns peer client identifier string
fn peer_info(&self, peer_id: PeerId) -> String {
peer_id.to_string()
}
/// Maximum mutuallt supported ETH protocol version
fn eth_protocol_version(&self, peer_id: PeerId) -> u8;
/// Returns if the chain block queue empty
fn is_chain_queue_empty(&self) -> bool {
self.chain().queue_info().is_empty()
@ -46,15 +52,17 @@ pub trait SyncIo {
/// Wraps `NetworkContext` and the blockchain client
pub struct NetSyncIo<'s, 'h> where 'h: 's {
network: &'s NetworkContext<'h>,
chain: &'s BlockChainClient
chain: &'s BlockChainClient,
snapshot_service: &'s SnapshotService,
}
impl<'s, 'h> NetSyncIo<'s, 'h> {
/// Creates a new instance from the `NetworkContext` and the blockchain client reference.
pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient) -> NetSyncIo<'s, 'h> {
pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient, snapshot_service: &'s SnapshotService) -> NetSyncIo<'s, 'h> {
NetSyncIo {
network: network,
chain: chain,
snapshot_service: snapshot_service,
}
}
}
@ -80,6 +88,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> {
self.chain
}
fn snapshot_service(&self) -> &SnapshotService {
self.snapshot_service
}
fn peer_info(&self, peer_id: PeerId) -> String {
self.network.peer_info(peer_id)
}
@ -87,6 +99,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> {
fn is_expired(&self) -> bool {
self.network.is_expired()
}
fn eth_protocol_version(&self, peer_id: PeerId) -> u8 {
self.network.protocol_version(peer_id, ETH_PROTOCOL).unwrap_or(0)
}
}

29
sync/src/tests/helpers.rs
View File

@ -16,22 +16,26 @@
use util::*;
use network::*;
use tests::snapshot::*;
use ethcore::client::{TestBlockChainClient, BlockChainClient};
use ethcore::header::BlockNumber;
use ethcore::snapshot::SnapshotService;
use sync_io::SyncIo;
use chain::ChainSync;
use ::SyncConfig;
pub struct TestIo<'p> {
pub chain: &'p mut TestBlockChainClient,
pub snapshot_service: &'p TestSnapshotService,
pub queue: &'p mut VecDeque<TestPacket>,
pub sender: Option<PeerId>,
}
impl<'p> TestIo<'p> {
pub fn new(chain: &'p mut TestBlockChainClient, queue: &'p mut VecDeque<TestPacket>, sender: Option<PeerId>) -> TestIo<'p> {
pub fn new(chain: &'p mut TestBlockChainClient, ss: &'p TestSnapshotService, queue: &'p mut VecDeque<TestPacket>, sender: Option<PeerId>) -> TestIo<'p> {
TestIo {
chain: chain,
snapshot_service: ss,
queue: queue,
sender: sender
}
@ -70,6 +74,14 @@ impl<'p> SyncIo for TestIo<'p> {
fn chain(&self) -> &BlockChainClient {
self.chain
}
fn snapshot_service(&self) -> &SnapshotService {
self.snapshot_service
}
fn eth_protocol_version(&self, _peer: PeerId) -> u8 {
64
}
}
pub struct TestPacket {
@ -80,6 +92,7 @@ pub struct TestPacket {
pub struct TestPeer {
pub chain: TestBlockChainClient,
pub snapshot_service: Arc<TestSnapshotService>,
pub sync: RwLock<ChainSync>,
pub queue: VecDeque<TestPacket>,
}
@ -103,9 +116,11 @@ impl TestNet {
let chain = TestBlockChainClient::new();
let mut config = SyncConfig::default();
config.fork_block = fork;
let ss = Arc::new(TestSnapshotService::new());
let sync = ChainSync::new(config, &chain);
net.peers.push(TestPeer {
sync: RwLock::new(sync),
snapshot_service: ss,
chain: chain,
queue: VecDeque::new(),
});
@ -126,7 +141,7 @@ impl TestNet {
for client in 0..self.peers.len() {
if peer != client {
let mut p = self.peers.get_mut(peer).unwrap();
p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(client as PeerId)), client as PeerId);
p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(client as PeerId)), client as PeerId);
}
}
}
@ -137,22 +152,22 @@ impl TestNet {
if let Some(packet) = self.peers[peer].queue.pop_front() {
let mut p = self.peers.get_mut(packet.recipient).unwrap();
trace!("--- {} -> {} ---", peer, packet.recipient);
ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data);
ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data);
trace!("----------------");
}
let mut p = self.peers.get_mut(peer).unwrap();
p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &mut p.queue, None));
p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, None));
}
}
pub fn sync_step_peer(&mut self, peer_num: usize) {
let mut peer = self.peer_mut(peer_num);
peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None));
peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None));
}
pub fn restart_peer(&mut self, i: usize) {
let peer = self.peer_mut(i);
peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None));
peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None));
}
pub fn sync(&mut self) -> u32 {
@ -181,6 +196,6 @@ impl TestNet {
pub fn trigger_chain_new_blocks(&mut self, peer_id: usize) {
let mut peer = self.peer_mut(peer_id);
peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None), &[], &[], &[], &[], &[]);
peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None), &[], &[], &[], &[], &[]);
}
}

1
sync/src/tests/mod.rs
View File

@ -15,5 +15,6 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
pub mod helpers;
pub mod snapshot;
mod chain;
mod rpc;

123
sync/src/tests/snapshot.rs Normal file
View File

@ -0,0 +1,123 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use util::*;
use ethcore::snapshot::{SnapshotService, ManifestData, RestorationStatus};
use ethcore::header::BlockNumber;
use ethcore::client::{EachBlockWith};
use super::helpers::*;
pub struct TestSnapshotService {
manifest: Option<ManifestData>,
chunks: HashMap<H256, Bytes>,
restoration_manifest: Mutex<Option<ManifestData>>,
state_restoration_chunks: Mutex<HashMap<H256, Bytes>>,
block_restoration_chunks: Mutex<HashMap<H256, Bytes>>,
}
impl TestSnapshotService {
pub fn new() -> TestSnapshotService {
TestSnapshotService {
manifest: None,
chunks: HashMap::new(),
restoration_manifest: Mutex::new(None),
state_restoration_chunks: Mutex::new(HashMap::new()),
block_restoration_chunks: Mutex::new(HashMap::new()),
}
}
pub fn new_with_snapshot(num_chunks: usize, block_hash: H256, block_number: BlockNumber) -> TestSnapshotService {
let num_state_chunks = num_chunks / 2;
let num_block_chunks = num_chunks - num_state_chunks;
let state_chunks: Vec<Bytes> = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect();
let block_chunks: Vec<Bytes> = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect();
let manifest = ManifestData {
state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
state_root: H256::new(),
block_number: block_number,
block_hash: block_hash,
};
let mut chunks: HashMap<H256, Bytes> = state_chunks.into_iter().map(|data| (data.sha3(), data)).collect();
chunks.extend(block_chunks.into_iter().map(|data| (data.sha3(), data)));
TestSnapshotService {
manifest: Some(manifest),
chunks: chunks,
restoration_manifest: Mutex::new(None),
state_restoration_chunks: Mutex::new(HashMap::new()),
block_restoration_chunks: Mutex::new(HashMap::new()),
}
}
}
impl SnapshotService for TestSnapshotService {
fn manifest(&self) -> Option<ManifestData> {
self.manifest.as_ref().cloned()
}
fn chunk(&self, hash: H256) -> Option<Bytes> {
self.chunks.get(&hash).cloned()
}
fn status(&self) -> RestorationStatus {
match &*self.restoration_manifest.lock() {
&Some(ref manifest) if self.state_restoration_chunks.lock().len() == manifest.state_hashes.len() &&
self.block_restoration_chunks.lock().len() == manifest.block_hashes.len() => RestorationStatus::Inactive,
&Some(_) => RestorationStatus::Ongoing {
state_chunks_done: self.state_restoration_chunks.lock().len() as u32,
block_chunks_done: self.block_restoration_chunks.lock().len() as u32,
},
&None => RestorationStatus::Inactive,
}
}
fn begin_restore(&self, manifest: ManifestData) {
*self.restoration_manifest.lock() = Some(manifest);
self.state_restoration_chunks.lock().clear();
self.block_restoration_chunks.lock().clear();
}
fn abort_restore(&self) {
*self.restoration_manifest.lock() = None;
self.state_restoration_chunks.lock().clear();
self.block_restoration_chunks.lock().clear();
}
fn restore_state_chunk(&self, hash: H256, chunk: Bytes) {
if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.state_hashes.iter().any(|h| h == &hash)) {
self.state_restoration_chunks.lock().insert(hash, chunk);
}
}
fn restore_block_chunk(&self, hash: H256, chunk: Bytes) {
if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.block_hashes.iter().any(|h| h == &hash)) {
self.block_restoration_chunks.lock().insert(hash, chunk);
}
}
}
#[test]
fn snapshot_sync() {
::env_logger::init().ok();
let mut net = TestNet::new(2);
net.peer_mut(0).snapshot_service = Arc::new(TestSnapshotService::new_with_snapshot(16, H256::new(), 1));
net.peer_mut(0).chain.add_blocks(1, EachBlockWith::Nothing);
net.sync_steps(19); // status + manifest + chunks
assert_eq!(net.peer(1).snapshot_service.state_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().state_hashes.len());
assert_eq!(net.peer(1).snapshot_service.block_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().block_hashes.len());
}

6
util/network/src/host.rs
View File

@ -282,6 +282,12 @@ impl<'s> NetworkContext<'s> {
}
"unknown".to_owned()
}
/// Returns max version for a given protocol.
pub fn protocol_version(&self, peer: PeerId, protocol: &str) -> Option<u8> {
let session = self.resolve_session(peer);
session.and_then(|s| s.lock().capability_version(protocol))
}
}
/// Shared host information

5
util/network/src/session.rs
View File

@ -243,6 +243,11 @@ impl Session {
self.info.capabilities.iter().any(|c| c.protocol == protocol)
}
/// Checks if peer supports given capability
pub fn capability_version(&self, protocol: &str) -> Option<u8> {
self.info.capabilities.iter().filter_map(|c| if c.protocol == protocol { Some(c.version) } else { None }).max()
}
/// Register the session socket with the event loop
pub fn register_socket<Host:Handler<Timeout = Token>>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), NetworkError> {
if self.expired() {

215
util/src/kvdb.rs
View File

@ -16,9 +16,11 @@
//! Key-Value store abstraction with `RocksDB` backend.
use std::io::ErrorKind;
use common::*;
use elastic_array::*;
use std::default::Default;
use std::path::PathBuf;
use rlp::{UntrustedRlp, RlpType, View, Compressible};
use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column};
@ -189,12 +191,18 @@ impl<'a> Iterator for DatabaseIterator {
}
}
struct DBAndColumns {
db: DB,
cfs: Vec<Column>,
}
/// Key-Value database.
pub struct Database {
db: DB,
db: RwLock<Option<DBAndColumns>>,
config: DatabaseConfig,
write_opts: WriteOptions,
cfs: Vec<Column>,
overlay: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
path: String,
}
impl Database {
@ -278,11 +286,13 @@ impl Database {
},
Err(s) => { return Err(s); }
};
let num_cols = cfs.len();
Ok(Database {
db: db,
db: RwLock::new(Some(DBAndColumns{ db: db, cfs: cfs })),
config: config.clone(),
write_opts: write_opts,
overlay: RwLock::new((0..(cfs.len() + 1)).map(|_| HashMap::new()).collect()),
cfs: cfs,
overlay: RwLock::new((0..(num_cols + 1)).map(|_| HashMap::new()).collect()),
path: path.to_owned(),
})
}
@ -320,94 +330,167 @@ impl Database {
/// Commit buffered changes to database.
pub fn flush(&self) -> Result<(), String> {
let batch = WriteBatch::new();
let mut overlay = self.overlay.write();
match &*self.db.read() {
&Some(DBAndColumns { ref db, ref cfs }) => {
let batch = WriteBatch::new();
let mut overlay = self.overlay.write();
for (c, column) in overlay.iter_mut().enumerate() {
let column_data = mem::replace(column, HashMap::new());
for (key, state) in column_data.into_iter() {
match state {
KeyState::Delete => {
if c > 0 {
try!(batch.delete_cf(self.cfs[c - 1], &key));
} else {
try!(batch.delete(&key));
}
},
KeyState::Insert(value) => {
if c > 0 {
try!(batch.put_cf(self.cfs[c - 1], &key, &value));
} else {
try!(batch.put(&key, &value));
}
},
KeyState::InsertCompressed(value) => {
let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
if c > 0 {
try!(batch.put_cf(self.cfs[c - 1], &key, &compressed));
} else {
try!(batch.put(&key, &value));
for (c, column) in overlay.iter_mut().enumerate() {
let column_data = mem::replace(column, HashMap::new());
for (key, state) in column_data.into_iter() {
match state {
KeyState::Delete => {
if c > 0 {
try!(batch.delete_cf(cfs[c - 1], &key));
} else {
try!(batch.delete(&key));
}
},
KeyState::Insert(value) => {
if c > 0 {
try!(batch.put_cf(cfs[c - 1], &key, &value));
} else {
try!(batch.put(&key, &value));
}
},
KeyState::InsertCompressed(value) => {
let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
if c > 0 {
try!(batch.put_cf(cfs[c - 1], &key, &compressed));
} else {
try!(batch.put(&key, &value));
}
}
}
}
}
}
db.write_opt(batch, &self.write_opts)
},
&None => Err("Database is closed".to_owned())
}
self.db.write_opt(batch, &self.write_opts)
}
/// Commit transaction to database.
pub fn write(&self, tr: DBTransaction) -> Result<(), String> {
let batch = WriteBatch::new();
let ops = tr.ops;
for op in ops {
match op {
DBOp::Insert { col, key, value } => {
try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(self.cfs[c as usize], &key, &value)))
},
DBOp::InsertCompressed { col, key, value } => {
let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(self.cfs[c as usize], &key, &compressed)))
},
DBOp::Delete { col, key } => {
try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(self.cfs[c as usize], &key)))
},
}
match &*self.db.read() {
&Some(DBAndColumns { ref db, ref cfs }) => {
let batch = WriteBatch::new();
let ops = tr.ops;
for op in ops {
match op {
DBOp::Insert { col, key, value } => {
try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(cfs[c as usize], &key, &value)))
},
DBOp::InsertCompressed { col, key, value } => {
let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(cfs[c as usize], &key, &compressed)))
},
DBOp::Delete { col, key } => {
try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(cfs[c as usize], &key)))
},
}
}
db.write_opt(batch, &self.write_opts)
},
&None => Err("Database is closed".to_owned())
}
self.db.write_opt(batch, &self.write_opts)
}
/// Get value by key.
pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<Bytes>, String> {
let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
match overlay.get(key) {
Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
Some(&KeyState::Delete) => Ok(None),
None => {
col.map_or_else(
|| self.db.get(key).map(|r| r.map(|v| v.to_vec())),
|c| self.db.get_cf(self.cfs[c as usize], key).map(|r| r.map(|v| v.to_vec())))
match &*self.db.read() {
&Some(DBAndColumns { ref db, ref cfs }) => {
let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
match overlay.get(key) {
Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
Some(&KeyState::Delete) => Ok(None),
None => {
col.map_or_else(
|| db.get(key).map(|r| r.map(|v| v.to_vec())),
|c| db.get_cf(cfs[c as usize], key).map(|r| r.map(|v| v.to_vec())))
},
}
},
&None => Ok(None),
}
}
/// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values.
// TODO: support prefix seek for unflushed ata
// TODO: support prefix seek for unflushed data
pub fn get_by_prefix(&self, col: Option<u32>, prefix: &[u8]) -> Option<Box<[u8]>> {
let mut iter = col.map_or_else(|| self.db.iterator(IteratorMode::From(prefix, Direction::Forward)),
|c| self.db.iterator_cf(self.cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
match iter.next() {
// TODO: use prefix_same_as_start read option (not availabele in C API currently)
Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
_ => None
match &*self.db.read() {
&Some(DBAndColumns { ref db, ref cfs }) => {
let mut iter = col.map_or_else(|| db.iterator(IteratorMode::From(prefix, Direction::Forward)),
|c| db.iterator_cf(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
match iter.next() {
// TODO: use prefix_same_as_start read option (not availabele in C API currently)
Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
_ => None
}
},
&None => None,
}
}
/// Get database iterator for flushed data.
pub fn iter(&self, col: Option<u32>) -> DatabaseIterator {
//TODO: iterate over overlay
col.map_or_else(|| DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) },
|c| DatabaseIterator { iter: self.db.iterator_cf(self.cfs[c as usize], IteratorMode::Start).unwrap() })
match &*self.db.read() {
&Some(DBAndColumns { ref db, ref cfs }) => {
col.map_or_else(|| DatabaseIterator { iter: db.iterator(IteratorMode::Start) },
|c| DatabaseIterator { iter: db.iterator_cf(cfs[c as usize], IteratorMode::Start).unwrap() })
},
&None => panic!("Not supported yet") //TODO: return an empty iterator or change return type
}
}
/// Close the database
fn close(&self) {
*self.db.write() = None;
self.overlay.write().clear();
}
/// Restore the database from a copy at given path.
pub fn restore(&self, new_db: &str) -> Result<(), UtilError> {
self.close();
let mut backup_db = PathBuf::from(&self.path);
backup_db.pop();
backup_db.push("backup_db");
println!("Path at {:?}", self.path);
println!("Backup at {:?}", backup_db);
let existed = match fs::rename(&self.path, &backup_db) {
Ok(_) => true,
Err(e) => if let ErrorKind::NotFound = e.kind() {
false
} else {
return Err(e.into());
}
};
match fs::rename(&new_db, &self.path) {
Ok(_) => {
// clean up the backup.
if existed {
try!(fs::remove_dir_all(&backup_db));
}
}
Err(e) => {
// restore the backup.
if existed {
try!(fs::rename(&backup_db, &self.path));
}
return Err(e.into())
}
}
// reopen the database and steal handles into self
let db = try!(Self::open(&self.config, &self.path));
*self.db.write() = mem::replace(&mut *db.db.write(), None);
*self.overlay.write() = mem::replace(&mut *db.overlay.write(), Vec::new());
Ok(())
}
}