Merge branch 'master' into dapp-norefresh

Conflicts:
	dapps/src/apps/fetcher.rs
	dapps/src/handlers/fetch.rs

.gitlab-ci.yml

@@ -9,6 +9,25 @@ variables:
 cache:
   key: "$CI_BUILD_NAME/$CI_BUILD_REF_NAME"
   untracked: true
+linux-stable:
+  stage: build
+  image: ethcore/rust:stable
+  only:
+    - master
+    - beta
+    - tags
+    - stable
+  script:
+    - export
+    - cargo build --release --verbose
+    - strip target/release/parity
+  tags:
+    - rust
+    - rust-stable
+  artifacts:
+    paths:
+    - target/release/parity
+    name: "${CI_BUILD_NAME}_parity"
 linux-beta:
   stage: build
   image: ethcore/rust:beta
@@ -29,12 +48,6 @@ linux-beta:
     paths:
     - target/release/parity
     name: "${CI_BUILD_NAME}_parity"
-  stage: deploy
-  tags:
-    - rust
-    - rust-beta
-  script:
-    - ./deploy.sh
 linux-nightly:
   stage: build
   image: ethcore/rust:nightly
@@ -84,6 +97,7 @@ linux-armv7:
     - stable
   script:
     - export
+    - export CXX="arm-linux-gnueabihf-g++"
     - rm -rf .cargo
     - mkdir -p .cargo
     - echo "[target.armv7-unknown-linux-gnueabihf]" >> .cargo/config
@@ -98,6 +112,7 @@ linux-armv7:
     paths:
     - target/armv7-unknown-linux-gnueabihf/release/parity
     name: "${CI_BUILD_NAME}_parity"
+  allow_failure: true
 linux-arm:
   stage: build
   image: ethcore/rust-arm:latest
@@ -108,11 +123,11 @@ linux-arm:
     - stable
   script:
     - export
-    - rm -rf .cargo
+    #- rm -rf .cargo
-    - mkdir -p .cargo
+    #- mkdir -p .cargo
-    - echo "[target.arm-unknown-linux-gnueabihf]" >> .cargo/config
+    #- echo "[target.arm-unknown-linux-gnueabihf]" >> .cargo/config
-    - echo "linker= \"arm-linux-gnueabihf-gcc\"" >> .cargo/config
+    #- echo "linker= \"arm-linux-gnueabihf-gcc\"" >> .cargo/config
-    - cat .cargo/config
+    #- cat .cargo/config
     - cargo build --target arm-unknown-linux-gnueabihf --release --verbose
     - arm-linux-gnueabihf-strip target/arm-unknown-linux-gnueabihf/release/parity
   tags:
@@ -133,11 +148,11 @@ linux-armv6:
     - stable
   script:
     - export
-    - rm -rf .cargo
+    #- rm -rf .cargo
-    - mkdir -p .cargo
+    #- mkdir -p .cargo
-    - echo "[target.arm-unknown-linux-gnueabi]" >> .cargo/config
+    #- echo "[target.arm-unknown-linux-gnueabi]" >> .cargo/config
-    - echo "linker= \"arm-linux-gnueabi-gcc\"" >> .cargo/config
+    #- echo "linker= \"arm-linux-gnueabi-gcc\"" >> .cargo/config
-    - cat .cargo/config
+    #- cat .cargo/config
     - cargo build --target arm-unknown-linux-gnueabi --release --verbose
     - arm-linux-gnueabi-strip target/arm-unknown-linux-gnueabi/release/parity
   tags:
@@ -158,11 +173,11 @@ linux-aarch64:
     - stable
   script:
     - export
-    - rm -rf .cargo
+    #- rm -rf .cargo
-    - mkdir -p .cargo
+    #- mkdir -p .cargo
-    - echo "[target.aarch64-unknown-linux-gnu]" >> .cargo/config
+    #- echo "[target.aarch64-unknown-linux-gnu]" >> .cargo/config
-    - echo "linker= \"aarch64-linux-gnu-gcc\"" >> .cargo/config
+    #- echo "linker= \"aarch64-linux-gnu-gcc\"" >> .cargo/config
-    - cat .cargo/config
+    #- cat .cargo/config
     - cargo build --target aarch64-unknown-linux-gnu --release --verbose
     - aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/parity
   tags:
@@ -208,30 +223,22 @@ windows:
     - target/release/parity.exe
     - target/release/parity.pdb
     name: "${CI_BUILD_NAME}_parity"
-linux-stable:
-  stage: build
-  image: ethcore/rust:stable
-  only:
-    - master
-    - beta
-    - tags
-    - stable
-  script:
-    - export
-    - cargo build --release --verbose
-    - strip target/release/parity
-  tags:
-    - rust
-    - rust-stable
-  artifacts:
-    paths:
-    - target/release/parity
-    name: "${CI_BUILD_NAME}_parity"
 test-linux:
   stage: test
   before_script:
     - git submodule update --init --recursive
   script:
     - ./test.sh --verbose
+  tags:
+    - rust-test
   dependencies:
     - linux-stable
+deploy-binaries:
+  stage: deploy
+  only:
+    - master
+    - beta
+    - tags
+    - stable
+  script:
+    - ./deploy.sh

Cargo.lock

@@ -390,6 +390,7 @@ name = "ethcore-ipc-nano"
 version = "1.4.0"
 dependencies = [
  "ethcore-ipc 1.4.0",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)",
 ]
@@ -569,6 +570,7 @@ name = "ethkey"
 version = "0.2.0"
 dependencies = [
  "bigint 0.1.0",
+ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)",
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -580,6 +582,7 @@ dependencies = [
 name = "ethstore"
 version = "0.1.0"
 dependencies = [
+ "docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcrypto 0.1.0",
  "ethkey 0.2.0",
  "itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -63,6 +63,8 @@ ipc = ["ethcore/ipc"]
 dev = ["clippy", "ethcore/dev", "ethcore-util/dev", "ethsync/dev", "ethcore-rpc/dev", "ethcore-dapps/dev", "ethcore-signer/dev"]
 json-tests = ["ethcore/json-tests"]
 stratum = ["ipc"]
+ethkey-cli = ["ethcore/ethkey-cli"]
+ethstore-cli = ["ethcore/ethstore-cli"]
 
 [[bin]]
 path = "parity/main.rs"

dapps/Cargo.toml

@@ -39,7 +39,7 @@ clippy = { version = "0.0.85", optional = true}
 serde_codegen = { version = "0.8", optional = true }
 
 [features]
-default = ["serde_codegen", "extra-dapps", "https-fetch/ca-github-only"]
+default = ["serde_codegen", "extra-dapps"]
 extra-dapps = ["parity-dapps-wallet"]
 nightly = ["serde_macros"]
 dev = ["clippy", "ethcore-rpc/dev", "ethcore-util/dev"]

dapps/src/apps/fetcher.rs

@@ -37,67 +37,67 @@ use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
 use endpoint::{Endpoint, EndpointPath, Handler};
 use apps::cache::{ContentCache, ContentStatus};
 use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
-use apps::urlhint::{URLHintContract, URLHint};
+use apps::urlhint::{URLHintContract, URLHint, URLHintResult};
 
 const MAX_CACHED_DAPPS: usize = 10;
 
-pub struct AppFetcher<R: URLHint = URLHintContract> {
+pub struct ContentFetcher<R: URLHint = URLHintContract> {
 	dapps_path: PathBuf,
 	resolver: R,
+	cache: Arc<Mutex<ContentCache>>,
 	sync: Arc<SyncStatus>,
-	dapps: Arc<Mutex<ContentCache>>,
 }
 
-impl<R: URLHint> Drop for AppFetcher<R> {
+impl<R: URLHint> Drop for ContentFetcher<R> {
 	fn drop(&mut self) {
 		// Clear cache path
 		let _ = fs::remove_dir_all(&self.dapps_path);
 	}
 }
 
-impl<R: URLHint> AppFetcher<R> {
+impl<R: URLHint> ContentFetcher<R> {
 
 	pub fn new(resolver: R, sync_status: Arc<SyncStatus>) -> Self {
 		let mut dapps_path = env::temp_dir();
 		dapps_path.push(random_filename());
 
-		AppFetcher {
+		ContentFetcher {
 			dapps_path: dapps_path,
 			resolver: resolver,
 			sync: sync_status,
-			dapps: Arc::new(Mutex::new(ContentCache::default())),
+			cache: Arc::new(Mutex::new(ContentCache::default())),
 		}
 	}
 
 	#[cfg(test)]
-	fn set_status(&self, app_id: &str, status: ContentStatus) {
+	fn set_status(&self, content_id: &str, status: ContentStatus) {
-		self.dapps.lock().insert(app_id.to_owned(), status);
+		self.cache.lock().insert(content_id.to_owned(), status);
 	}
 
-	pub fn contains(&self, app_id: &str) -> bool {
+	pub fn contains(&self, content_id: &str) -> bool {
 		{
-			let mut dapps = self.dapps.lock();
+			let mut cache = self.cache.lock();
 			// Check if we already have the app
-			if dapps.get(app_id).is_some() {
+			if cache.get(content_id).is_some() {
 				return true;
 			}
 		}
 		// fallback to resolver
-		if let Ok(app_id) = app_id.from_hex() {
+		if let Ok(content_id) = content_id.from_hex() {
 			// if app_id is valid, but we are syncing always return true.
 			if self.sync.is_major_syncing() {
 				return true;
 			}
 			// else try to resolve the app_id
-			self.resolver.resolve(app_id).is_some()
+			self.resolver.resolve(content_id).is_some()
 		} else {
 			false
 		}
 	}
 
 	pub fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
-		let mut dapps = self.dapps.lock();
+		let mut cache = self.cache.lock();
-		let app_id = path.app_id.clone();
+		let content_id = path.app_id.clone();
 
 		if self.sync.is_major_syncing() {
 			return Box::new(ContentHandler::error(
@@ -109,7 +109,7 @@ impl<R: URLHint> AppFetcher<R> {
 		}
 
 		let (new_status, handler) = {
-			let status = dapps.get(&app_id);
+			let status = cache.get(&content_id);
 			match status {
 				// Just server dapp
 				Some(&mut ContentStatus::Ready(ref endpoint)) => {
@@ -122,40 +122,72 @@ impl<R: URLHint> AppFetcher<R> {
 				},
 				// We need to start fetching app
 				None => {
-					trace!(target: "dapps", "Content fetching unavailable. Fetching...");
+					trace!(target: "dapps", "Content unavailable. Fetching...");
-					let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true.");
+					let content_hex = content_id.from_hex().expect("to_handler is called only when `contains` returns true.");
-					let app = self.resolver.resolve(app_hex);
+					let content = self.resolver.resolve(content_hex);
 
-					if let Some(app) = app {
+					let cache = self.cache.clone();
-						let (handler, fetch_control) = ContentFetcherHandler::new(
+					let on_done = move |id: String, result: Option<LocalPageEndpoint>| {
-							app,
+						let mut cache = cache.lock();
-							control,
+						match result {
-							path.using_dapps_domains,
+							Some(endpoint) => {
-							DappInstaller {
+								cache.insert(id, ContentStatus::Ready(endpoint));
-								dapp_id: app_id.clone(),
+							},
-								dapps_path: self.dapps_path.clone(),
+							// In case of error
-								dapps: self.dapps.clone(),
+							None => {
-							}
+								cache.remove(&id);
-						);
+							},
+						}
+					};
 
-						(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
+					match content {
-					} else {
+						Some(URLHintResult::Dapp(dapp)) => {
-						// This may happen when sync status changes in between
+							let (handler, fetch_control) = ContentFetcherHandler::new(
-						// `contains` and `to_handler`
+								dapp.url(),
-						(None, Box::new(ContentHandler::error(
+								control,
-							StatusCode::NotFound,
+								path.using_dapps_domains,
-							"Resource Not Found",
+								DappInstaller {
-							"Requested resource was not found.",
+									id: content_id.clone(),
-							None
+									dapps_path: self.dapps_path.clone(),
-						)) as Box<Handler>)
+									on_done: Box::new(on_done),
+								}
+							);
+
+							(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
+						},
+						Some(URLHintResult::Content(content)) => {
+							let (handler, fetch_control) = ContentFetcherHandler::new(
+								content.url,
+								control,
+								path.using_dapps_domains,
+								ContentInstaller {
+									id: content_id.clone(),
+									mime: content.mime,
+									content_path: self.dapps_path.clone(),
+									on_done: Box::new(on_done),
+								}
+							);
+
+							(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
+						},
+						None => {
+							// This may happen when sync status changes in between
+							// `contains` and `to_handler`
+							(None, Box::new(ContentHandler::error(
+								StatusCode::NotFound,
+								"Resource Not Found",
+								"Requested resource was not found.",
+								None
+							)) as Box<Handler>)
+						},
 					}
 				},
 			}
 		};
 
 		if let Some(status) = new_status {
-			dapps.clear_garbage(MAX_CACHED_DAPPS);
+			cache.clear_garbage(MAX_CACHED_DAPPS);
-			dapps.insert(app_id, status);
+			cache.insert(content_id, status);
 		}
 
 		handler
@@ -166,7 +198,7 @@ impl<R: URLHint> AppFetcher<R> {
 pub enum ValidationError {
 	Io(io::Error),
 	Zip(zip::result::ZipError),
-	InvalidDappId,
+	InvalidContentId,
 	ManifestNotFound,
 	ManifestSerialization(String),
 	HashMismatch { expected: H256, got: H256, },
@@ -177,7 +209,7 @@ impl fmt::Display for ValidationError {
 		match *self {
 			ValidationError::Io(ref io) => write!(f, "Unexpected IO error occured: {:?}", io),
 			ValidationError::Zip(ref zip) => write!(f, "Unable to read ZIP archive: {:?}", zip),
-			ValidationError::InvalidDappId => write!(f, "Dapp ID is invalid. It should be 32 bytes hash of content."),
+			ValidationError::InvalidContentId => write!(f, "ID is invalid. It should be 256 bits keccak hash of content."),
 			ValidationError::ManifestNotFound => write!(f, "Downloaded Dapp bundle did not contain valid manifest.json file."),
 			ValidationError::ManifestSerialization(ref err) => {
 				write!(f, "There was an error during Dapp Manifest serialization: {:?}", err)
@@ -201,10 +233,44 @@ impl From<zip::result::ZipError> for ValidationError {
 	}
 }
 
+struct ContentInstaller {
+	id: String,
+	mime: String,
+	content_path: PathBuf,
+	on_done: Box<Fn(String, Option<LocalPageEndpoint>) + Send>,
+}
+
+impl ContentValidator for ContentInstaller {
+	type Error = ValidationError;
+
+	fn validate_and_install(&self, path: PathBuf) -> Result<(String, LocalPageEndpoint), ValidationError> {
+		// Create dir
+		try!(fs::create_dir_all(&self.content_path));
+
+		// And prepare path for a file
+		let filename = path.file_name().expect("We always fetch a file.");
+		let mut content_path = self.content_path.clone();
+		content_path.push(&filename);
+
+		if content_path.exists() {
+			try!(fs::remove_dir_all(&content_path))
+		}
+
+		try!(fs::copy(&path, &content_path));
+
+		Ok((self.id.clone(), LocalPageEndpoint::single_file(content_path, self.mime.clone())))
+	}
+
+	fn done(&self, endpoint: Option<LocalPageEndpoint>) {
+		(self.on_done)(self.id.clone(), endpoint)
+	}
+}
+
+
 struct DappInstaller {
-	dapp_id: String,
+	id: String,
 	dapps_path: PathBuf,
-	dapps: Arc<Mutex<ContentCache>>,
+	on_done: Box<Fn(String, Option<LocalPageEndpoint>) + Send>,
 }
 
 impl DappInstaller {
@@ -242,14 +308,14 @@ impl DappInstaller {
 impl ContentValidator for DappInstaller {
 	type Error = ValidationError;
 
-	fn validate_and_install(&self, app_path: PathBuf) -> Result<Manifest, ValidationError> {
+	fn validate_and_install(&self, app_path: PathBuf) -> Result<(String, LocalPageEndpoint), ValidationError> {
 		trace!(target: "dapps", "Opening dapp bundle at {:?}", app_path);
 		let mut file_reader = io::BufReader::new(try!(fs::File::open(app_path)));
 		let hash = try!(sha3(&mut file_reader));
-		let dapp_id = try!(self.dapp_id.as_str().parse().map_err(|_| ValidationError::InvalidDappId));
+		let id = try!(self.id.as_str().parse().map_err(|_| ValidationError::InvalidContentId));
-		if dapp_id != hash {
+		if id != hash {
 			return Err(ValidationError::HashMismatch {
-				expected: dapp_id,
+				expected: id,
 				got: hash,
 			});
 		}
@@ -259,7 +325,7 @@ impl ContentValidator for DappInstaller {
 		// First find manifest file
 		let (mut manifest, manifest_dir) = try!(Self::find_manifest(&mut zip));
 		// Overwrite id to match hash
-		manifest.id = self.dapp_id.clone();
+		manifest.id = self.id.clone();
 
 		let target = self.dapp_target_path(&manifest);
 
@@ -296,23 +362,15 @@ impl ContentValidator for DappInstaller {
 		let mut manifest_file = try!(fs::File::create(manifest_path));
 		try!(manifest_file.write_all(manifest_str.as_bytes()));
 
+		// Create endpoint
+		let app = LocalPageEndpoint::new(target, manifest.clone().into());
+
 		// Return modified app manifest
-		Ok(manifest)
+		Ok((manifest.id.clone(), app))
 	}
 
-	fn done(&self, manifest: Option<&Manifest>) {
+	fn done(&self, endpoint: Option<LocalPageEndpoint>) {
-		let mut dapps = self.dapps.lock();
+		(self.on_done)(self.id.clone(), endpoint)
-		match manifest {
-			Some(manifest) => {
-				let path = self.dapp_target_path(manifest);
-				let app = LocalPageEndpoint::new(path, manifest.clone().into());
-				dapps.insert(self.dapp_id.clone(), ContentStatus::Ready(app));
-			},
-			// In case of error
-			None => {
-				dapps.remove(&self.dapp_id);
-			},
-		}
 	}
 }
 
@@ -324,12 +382,12 @@ mod tests {
 	use endpoint::EndpointInfo;
 	use page::LocalPageEndpoint;
 	use apps::cache::ContentStatus;
-	use apps::urlhint::{GithubApp, URLHint};
+	use apps::urlhint::{URLHint, URLHintResult};
-	use super::AppFetcher;
+	use super::ContentFetcher;
 
 	struct FakeResolver;
 	impl URLHint for FakeResolver {
-		fn resolve(&self, _app_id: Bytes) -> Option<GithubApp> {
+		fn resolve(&self, _id: Bytes) -> Option<URLHintResult> {
 			None
 		}
 	}
@@ -338,7 +396,7 @@ mod tests {
 	fn should_true_if_contains_the_app() {
 		// given
 		let path = env::temp_dir();
-		let fetcher = AppFetcher::new(FakeResolver, Arc::new(|| false));
+		let fetcher = ContentFetcher::new(FakeResolver, Arc::new(|| false));
 		let handler = LocalPageEndpoint::new(path, EndpointInfo {
 			name: "fake".into(),
 			description: "".into(),

dapps/src/apps/urlhint.rs

@@ -17,6 +17,7 @@
 use std::fmt;
 use std::sync::Arc;
 use rustc_serialize::hex::ToHex;
+use mime_guess;
 
 use ethabi::{Interface, Contract, Token};
 use util::{Address, Bytes, Hashable};
@@ -52,6 +53,13 @@ impl GithubApp {
 	}
 }
 
+#[derive(Debug, PartialEq)]
+pub struct Content {
+	pub url: String,
+	pub mime: String,
+	pub owner: Address,
+}
+
 /// RAW Contract interface.
 /// Should execute transaction using current blockchain state.
 pub trait ContractClient: Send + Sync {
@@ -61,10 +69,19 @@ pub trait ContractClient: Send + Sync {
 	fn call(&self, address: Address, data: Bytes) -> Result<Bytes, String>;
 }
 
+/// Result of resolving id to URL
+#[derive(Debug, PartialEq)]
+pub enum URLHintResult {
+	/// Dapp
+	Dapp(GithubApp),
+	/// Content
+	Content(Content),
+}
+
 /// URLHint Contract interface
 pub trait URLHint {
 	/// Resolves given id to registrar entry.
-	fn resolve(&self, app_id: Bytes) -> Option<GithubApp>;
+	fn resolve(&self, id: Bytes) -> Option<URLHintResult>;
 }
 
 pub struct URLHintContract {
@@ -110,10 +127,10 @@ impl URLHintContract {
 		}
 	}
 
-	fn encode_urlhint_call(&self, app_id: Bytes) -> Option<Bytes> {
+	fn encode_urlhint_call(&self, id: Bytes) -> Option<Bytes> {
 		let call = self.urlhint
 			.function("entries".into())
-			.and_then(|f| f.encode_call(vec![Token::FixedBytes(app_id)]));
+			.and_then(|f| f.encode_call(vec![Token::FixedBytes(id)]));
 
 		match call {
 			Ok(res) => {
@@ -126,7 +143,7 @@ impl URLHintContract {
 		}
 	}
 
-	fn decode_urlhint_output(&self, output: Bytes) -> Option<GithubApp> {
+	fn decode_urlhint_output(&self, output: Bytes) -> Option<URLHintResult> {
 		trace!(target: "dapps", "Output: {:?}", output.to_hex());
 		let output = self.urlhint
 			.function("entries".into())
@@ -149,6 +166,17 @@ impl URLHintContract {
 					if owner == Address::default() {
 						return None;
 					}
+
+					let commit = GithubApp::commit(&commit);
+					if commit == Some(Default::default()) {
+						let mime = guess_mime_type(&account_slash_repo).unwrap_or("application/octet-stream".into());
+						return Some(URLHintResult::Content(Content {
+							url: account_slash_repo,
+							mime: mime,
+							owner: owner,
+						}));
+					}
+
 					let (account, repo) = {
 						let mut it = account_slash_repo.split('/');
 						match (it.next(), it.next()) {
@@ -157,12 +185,12 @@ impl URLHintContract {
 						}
 					};
 
-					GithubApp::commit(&commit).map(|commit| GithubApp {
+					commit.map(|commit| URLHintResult::Dapp(GithubApp {
 						account: account,
 						repo: repo,
 						commit: commit,
 						owner: owner,
-					})
+					}))
 				},
 				e => {
 					warn!(target: "dapps", "Invalid contract output parameters: {:?}", e);
@@ -177,10 +205,10 @@ impl URLHintContract {
 }
 
 impl URLHint for URLHintContract {
-	fn resolve(&self, app_id: Bytes) -> Option<GithubApp> {
+	fn resolve(&self, id: Bytes) -> Option<URLHintResult> {
 		self.urlhint_address().and_then(|address| {
 			// Prepare contract call
-			self.encode_urlhint_call(app_id)
+			self.encode_urlhint_call(id)
 				.and_then(|data| {
 					let call = self.client.call(address, data);
 					if let Err(ref e) = call {
@@ -193,6 +221,34 @@ impl URLHint for URLHintContract {
 	}
 }
 
+fn guess_mime_type(url: &str) -> Option<String> {
+	const CONTENT_TYPE: &'static str = "content-type=";
+
+	let mut it = url.split('#');
+	// skip url
+	let url = it.next();
+	// get meta headers
+	let metas = it.next();
+	if let Some(metas) = metas {
+		for meta in metas.split('&') {
+			let meta = meta.to_lowercase();
+			if meta.starts_with(CONTENT_TYPE) {
+				return Some(meta[CONTENT_TYPE.len()..].to_owned());
+			}
+		}
+	}
+	url.and_then(|url| {
+		url.split('.').last()
+	}).and_then(|extension| {
+		mime_guess::get_mime_type_str(extension).map(Into::into)
+	})
+}
+
+#[cfg(test)]
+pub fn test_guess_mime_type(url: &str) -> Option<String> {
+	guess_mime_type(url)
+}
+
 fn as_string<T: fmt::Debug>(e: T) -> String {
 	format!("{:?}", e)
 }
@@ -201,7 +257,7 @@ fn as_string<T: fmt::Debug>(e: T) -> String {
 mod tests {
 	use std::sync::Arc;
 	use std::str::FromStr;
-	use rustc_serialize::hex::{ToHex, FromHex};
+	use rustc_serialize::hex::FromHex;
 
 	use super::*;
 	use util::{Bytes, Address, Mutex, ToPretty};
@@ -279,12 +335,33 @@ mod tests {
 		let res = urlhint.resolve("test".bytes().collect());
 
 		// then
-		assert_eq!(res, Some(GithubApp {
+		assert_eq!(res, Some(URLHintResult::Dapp(GithubApp {
 			account: "ethcore".into(),
 			repo: "dao.claim".into(),
 			commit: GithubApp::commit(&"ec4c1fe06c808fe3739858c347109b1f5f1ed4b5".from_hex().unwrap()).unwrap(),
 			owner: Address::from_str("deadcafebeefbeefcafedeaddeedfeedffffffff").unwrap(),
-		}))
+		})))
+	}
+
+	#[test]
+	fn should_decode_urlhint_content_output() {
+		// given
+		let mut registrar = FakeRegistrar::new();
+		registrar.responses = Mutex::new(vec![
+			Ok(format!("000000000000000000000000{}", URLHINT).from_hex().unwrap()),
+			Ok("00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000deadcafebeefbeefcafedeaddeedfeedffffffff000000000000000000000000000000000000000000000000000000000000003d68747470733a2f2f657468636f72652e696f2f6173736574732f696d616765732f657468636f72652d626c61636b2d686f72697a6f6e74616c2e706e67000000".from_hex().unwrap()),
+		]);
+		let urlhint = URLHintContract::new(Arc::new(registrar));
+
+		// when
+		let res = urlhint.resolve("test".bytes().collect());
+
+		// then
+		assert_eq!(res, Some(URLHintResult::Content(Content {
+			url: "https://ethcore.io/assets/images/ethcore-black-horizontal.png".into(),
+			mime: "image/png".into(),
+			owner: Address::from_str("deadcafebeefbeefcafedeaddeedfeedffffffff").unwrap(),
+		})))
 	}
 
 	#[test]
@@ -303,4 +380,20 @@ mod tests {
 		// then
 		assert_eq!(url, "https://codeload.github.com/test/xyz/zip/000102030405060708090a0b0c0d0e0f10111213".to_owned());
 	}
+
+	#[test]
+	fn should_guess_mime_type_from_url() {
+		let url1 = "https://ethcore.io/parity";
+		let url2 = "https://ethcore.io/parity#content-type=image/png";
+		let url3 = "https://ethcore.io/parity#something&content-type=image/png";
+		let url4 = "https://ethcore.io/parity.png#content-type=image/jpeg";
+		let url5 = "https://ethcore.io/parity.png";
+
+
+		assert_eq!(test_guess_mime_type(url1), None);
+		assert_eq!(test_guess_mime_type(url2), Some("image/png".into()));
+		assert_eq!(test_guess_mime_type(url3), Some("image/png".into()));
+		assert_eq!(test_guess_mime_type(url4), Some("image/jpeg".into()));
+		assert_eq!(test_guess_mime_type(url5), Some("image/png".into()));
+	}
 }

dapps/src/handlers/client/mod.rs

@@ -63,7 +63,7 @@ impl Client {
 		self.https_client.close();
 	}
 
-	pub fn request(&mut self, url: String, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
+	pub fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
 		let is_https = url.starts_with("https://");
 		let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
 		trace!(target: "dapps", "Fetching from: {:?}", url);

dapps/src/handlers/fetch.rs

@@ -30,23 +30,22 @@ use hyper::status::StatusCode;
 use handlers::{ContentHandler, Redirection};
 use handlers::client::{Client, FetchResult};
 use apps::redirection_address;
-use apps::urlhint::GithubApp;
+use page::LocalPageEndpoint;
-use apps::manifest::Manifest;
 
 const FETCH_TIMEOUT: u64 = 30;
 
 enum FetchState {
-	NotStarted(GithubApp),
+	NotStarted(String),
-	InProgress(mpsc::Receiver<FetchResult>),
 	Error(ContentHandler),
-	Done(Manifest, Redirection),
+	InProgress(mpsc::Receiver<FetchResult>),
+	Done(String, LocalPageEndpoint, Redirection),
 }
 
 pub trait ContentValidator {
 	type Error: fmt::Debug + fmt::Display;
 
-	fn validate_and_install(&self, app: PathBuf) -> Result<Manifest, Self::Error>;
+	fn validate_and_install(&self, app: PathBuf) -> Result<(String, LocalPageEndpoint), Self::Error>;
-	fn done(&self, Option<&Manifest>);
+	fn done(&self, Option<LocalPageEndpoint>);
 }
 
 pub struct FetchControl {
@@ -80,7 +79,7 @@ impl FetchControl {
 	fn set_status(&self, status: &FetchState) {
 		match *status {
 			FetchState::Error(ref handler) => self.notify(|| FetchState::Error(handler.clone())),
-			FetchState::Done(ref manifest, ref handler) => self.notify(|| FetchState::Done(manifest.clone(), handler.clone())),
+			FetchState::Done(ref id, ref endpoint, ref handler) => self.notify(|| FetchState::Done(id.clone(), endpoint.clone(), handler.clone())),
 			FetchState::NotStarted(_) | FetchState::InProgress(_) => {},
 		}
 	}
@@ -117,7 +116,7 @@ impl server::Handler<HttpStream> for WaitingHandler {
 
 	fn on_response(&mut self, res: &mut server::Response) -> Next {
 		match self.state {
-			Some(FetchState::Done(_, ref mut handler)) => handler.on_response(res),
+			Some(FetchState::Done(_, _, ref mut handler)) => handler.on_response(res),
 			Some(FetchState::Error(ref mut handler)) => handler.on_response(res),
 			_ => Next::end(),
 		}
@@ -125,7 +124,7 @@ impl server::Handler<HttpStream> for WaitingHandler {
 
 	fn on_response_writable(&mut self, encoder: &mut Encoder<HttpStream>) -> Next {
 		match self.state {
-			Some(FetchState::Done(_, ref mut handler)) => handler.on_response_writable(encoder),
+			Some(FetchState::Done(_, _, ref mut handler)) => handler.on_response_writable(encoder),
 			Some(FetchState::Error(ref mut handler)) => handler.on_response_writable(encoder),
 			_ => Next::end(),
 		}
@@ -134,27 +133,27 @@ impl server::Handler<HttpStream> for WaitingHandler {
 
 pub struct ContentFetcherHandler<H: ContentValidator> {
 	fetch_control: Arc<FetchControl>,
-	status: FetchState,
 	control: Option<Control>,
+	status: FetchState,
 	client: Option<Client>,
 	using_dapps_domains: bool,
-	dapp: H,
+	installer: H,
 }
 
 impl<H: ContentValidator> Drop for ContentFetcherHandler<H> {
 	fn drop(&mut self) {
-		let manifest = match self.status {
+		let result = match self.status {
-			FetchState::Done(ref manifest, _) => Some(manifest),
+			FetchState::Done(_, ref result, _) => Some(result.clone()),
 			_ => None,
 		};
-		self.dapp.done(manifest);
+		self.installer.done(result);
 	}
 }
 
 impl<H: ContentValidator> ContentFetcherHandler<H> {
 
 	pub fn new(
-		app: GithubApp,
+		url: String,
 		control: Control,
 		using_dapps_domains: bool,
 		handler: H) -> (Self, Arc<FetchControl>) {
@@ -165,9 +164,9 @@ impl<H: ContentValidator> ContentFetcherHandler<H> {
 			fetch_control: fetch_control.clone(),
 			control: Some(control),
 			client: Some(client),
-			status: FetchState::NotStarted(app),
+			status: FetchState::NotStarted(url),
 			using_dapps_domains: using_dapps_domains,
-			dapp: handler,
+			installer: handler,
 		};
 
 		(handler, fetch_control)
@@ -179,26 +178,25 @@ impl<H: ContentValidator> ContentFetcherHandler<H> {
 			.close();
 	}
 
-	fn fetch_app(client: &mut Client, app: &GithubApp, abort: Arc<AtomicBool>, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
+	fn fetch_content(client: &mut Client, url: &str, abort: Arc<AtomicBool>, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
-		let res = client.request(app.url(), abort, Box::new(move || {
+		client.request(url, abort, Box::new(move || {
 			trace!(target: "dapps", "Fetching finished.");
 			// Ignoring control errors
 			let _ = control.ready(Next::read());
-		})).map_err(|e| format!("{:?}", e));
+		})).map_err(|e| format!("{:?}", e))
-		res
 	}
 }
 
 impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<H> {
 	fn on_request(&mut self, request: server::Request<HttpStream>) -> Next {
-		let status = if let FetchState::NotStarted(ref app) = self.status {
+		let status = if let FetchState::NotStarted(ref url) = self.status {
 			Some(match *request.method() {
 				// Start fetching content
 				Method::Get => {
-					trace!(target: "dapps", "Fetching dapp: {:?}", app);
+					trace!(target: "dapps", "Fetching content from: {:?}", url);
 					let control = self.control.take().expect("on_request is called only once, thus control is always Some");
 					let client = self.client.as_mut().expect("on_request is called before client is closed.");
-					let fetch = Self::fetch_app(client, app, self.fetch_control.abort.clone(), control);
+					let fetch = Self::fetch_content(client, url, self.fetch_control.abort.clone(), control);
 					match fetch {
 						Ok(receiver) => FetchState::InProgress(receiver),
 						Err(e) => FetchState::Error(ContentHandler::error(
@@ -235,7 +233,7 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
 				let timeout = ContentHandler::error(
 					StatusCode::GatewayTimeout,
 					"Download Timeout",
-					&format!("Could not fetch dapp bundle within {} seconds.", FETCH_TIMEOUT),
+					&format!("Could not fetch content within {} seconds.", FETCH_TIMEOUT),
 					None
 				);
 				Self::close_client(&mut self.client);
@@ -247,22 +245,22 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
 				match rec {
 					// Unpack and validate
 					Ok(Ok(path)) => {
-						trace!(target: "dapps", "Fetching dapp finished. Starting validation.");
+						trace!(target: "dapps", "Fetching content finished. Starting validation ({:?})", path);
 						Self::close_client(&mut self.client);
 						// Unpack and verify
-						let state = match self.dapp.validate_and_install(path.clone()) {
+						let state = match self.installer.validate_and_install(path.clone()) {
 							Err(e) => {
-								trace!(target: "dapps", "Error while validating dapp: {:?}", e);
+								trace!(target: "dapps", "Error while validating content: {:?}", e);
 								FetchState::Error(ContentHandler::error(
 									StatusCode::BadGateway,
 									"Invalid Dapp",
-									"Downloaded bundle does not contain a valid dapp.",
+									"Downloaded bundle does not contain a valid content.",
 									Some(&format!("{:?}", e))
 								))
 							},
-							Ok(manifest) => {
+							Ok((id, result)) => {
-								let address = redirection_address(self.using_dapps_domains, &manifest.id);
+								let address = redirection_address(self.using_dapps_domains, &id);
-								FetchState::Done(manifest, Redirection::new(&address))
+								FetchState::Done(id, result, Redirection::new(&address))
 							},
 						};
 						// Remove temporary zip file
@@ -270,11 +268,11 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
 						(Some(state), Next::write())
 					},
 					Ok(Err(e)) => {
-						warn!(target: "dapps", "Unable to fetch new dapp: {:?}", e);
+						warn!(target: "dapps", "Unable to fetch content: {:?}", e);
 						let error = ContentHandler::error(
 							StatusCode::BadGateway,
 							"Download Error",
-							"There was an error when fetching the dapp.",
+							"There was an error when fetching the content.",
 							Some(&format!("{:?}", e)),
 						);
 						(Some(FetchState::Error(error)), Next::write())
@@ -297,7 +295,7 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
 
 	fn on_response(&mut self, res: &mut server::Response) -> Next {
 		match self.status {
-			FetchState::Done(_, ref mut handler) => handler.on_response(res),
+			FetchState::Done(_, _, ref mut handler) => handler.on_response(res),
 			FetchState::Error(ref mut handler) => handler.on_response(res),
 			_ => Next::end(),
 		}
@@ -305,7 +303,7 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
 
 	fn on_response_writable(&mut self, encoder: &mut Encoder<HttpStream>) -> Next {
 		match self.status {
-			FetchState::Done(_, ref mut handler) => handler.on_response_writable(encoder),
+			FetchState::Done(_, _, ref mut handler) => handler.on_response_writable(encoder),
 			FetchState::Error(ref mut handler) => handler.on_response_writable(encoder),
 			_ => Next::end(),
 		}

dapps/src/lib.rs

@@ -191,7 +191,7 @@ impl Server {
 	) -> Result<Server, ServerError> {
 		let panic_handler = Arc::new(Mutex::new(None));
 		let authorization = Arc::new(authorization);
-		let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status));
+		let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status));
 		let endpoints = Arc::new(apps::all_endpoints(dapps_path));
 		let special = Arc::new({
 			let mut special = HashMap::new();
@@ -206,7 +206,7 @@ impl Server {
 			.handle(move |ctrl| router::Router::new(
 				ctrl,
 				apps::main_page(),
-				apps_fetcher.clone(),
+				content_fetcher.clone(),
 				endpoints.clone(),
 				special.clone(),
 				authorization.clone(),

dapps/src/page/local.rs

@@ -17,20 +17,31 @@
 use mime_guess;
 use std::io::{Seek, Read, SeekFrom};
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use page::handler;
 use endpoint::{Endpoint, EndpointInfo, EndpointPath, Handler};
 
+#[derive(Debug, Clone)]
 pub struct LocalPageEndpoint {
 	path: PathBuf,
-	info: EndpointInfo,
+	mime: Option<String>,
+	info: Option<EndpointInfo>,
 }
 
 impl LocalPageEndpoint {
 	pub fn new(path: PathBuf, info: EndpointInfo) -> Self {
 		LocalPageEndpoint {
 			path: path,
-			info: info,
+			mime: None,
+			info: Some(info),
+		}
+	}
+
+	pub fn single_file(path: PathBuf, mime: String) -> Self {
+		LocalPageEndpoint {
+			path: path,
+			mime: Some(mime),
+			info: None,
 		}
 	}
 
@@ -41,17 +52,40 @@ impl LocalPageEndpoint {
 
 impl Endpoint for LocalPageEndpoint {
 	fn info(&self) -> Option<&EndpointInfo> {
-		Some(&self.info)
+		self.info.as_ref()
 	}
 
 	fn to_handler(&self, path: EndpointPath) -> Box<Handler> {
-		Box::new(handler::PageHandler {
+		if let Some(ref mime) = self.mime {
-			app: LocalDapp::new(self.path.clone()),
+			Box::new(handler::PageHandler {
-			prefix: None,
+				app: LocalSingleFile { path: self.path.clone(), mime: mime.clone() },
-			path: path,
+				prefix: None,
-			file: Default::default(),
+				path: path,
-			safe_to_embed: false,
+				file: Default::default(),
-		})
+				safe_to_embed: false,
+			})
+		} else {
+			Box::new(handler::PageHandler {
+				app: LocalDapp { path: self.path.clone() },
+				prefix: None,
+				path: path,
+				file: Default::default(),
+				safe_to_embed: false,
+			})
+		}
+	}
+}
+
+struct LocalSingleFile {
+	path: PathBuf,
+	mime: String,
+}
+
+impl handler::Dapp for LocalSingleFile {
+	type DappFile = LocalFile;
+
+	fn file(&self, _path: &str) -> Option<Self::DappFile> {
+		LocalFile::from_path(&self.path, Some(&self.mime))
 	}
 }
 
@@ -59,14 +93,6 @@ struct LocalDapp {
 	path: PathBuf,
 }
 
-impl LocalDapp {
-	fn new(path: PathBuf) -> Self {
-		LocalDapp {
-			path: path
-		}
-	}
-}
-
 impl handler::Dapp for LocalDapp {
 	type DappFile = LocalFile;
 
@@ -75,18 +101,7 @@ impl handler::Dapp for LocalDapp {
 		for part in file_path.split('/') {
 			path.push(part);
 		}
-		// Check if file exists
+		LocalFile::from_path(&path, None)
-		fs::File::open(path.clone()).ok().map(|file| {
-			let content_type = mime_guess::guess_mime_type(path);
-			let len = file.metadata().ok().map_or(0, |meta| meta.len());
-			LocalFile {
-				content_type: content_type.to_string(),
-				buffer: [0; 4096],
-				file: file,
-				pos: 0,
-				len: len,
-			}
-		})
 	}
 }
 
@@ -98,6 +113,24 @@ struct LocalFile {
 	pos: u64,
 }
 
+impl LocalFile {
+	fn from_path<P: AsRef<Path>>(path: P, mime: Option<&str>) -> Option<Self> {
+		// Check if file exists
+		fs::File::open(&path).ok().map(|file| {
+			let content_type = mime.map(|mime| mime.to_owned())
+				.unwrap_or_else(|| mime_guess::guess_mime_type(path).to_string());
+			let len = file.metadata().ok().map_or(0, |meta| meta.len());
+			LocalFile {
+				content_type: content_type,
+				buffer: [0; 4096],
+				file: file,
+				pos: 0,
+				len: len,
+			}
+		})
+	}
+}
+
 impl handler::DappFile for LocalFile {
 	fn content_type(&self) -> &str {
 		&self.content_type

dapps/src/router/mod.rs

@@ -27,7 +27,7 @@ use url::{Url, Host};
 use hyper::{self, server, Next, Encoder, Decoder, Control, StatusCode};
 use hyper::net::HttpStream;
 use apps;
-use apps::fetcher::AppFetcher;
+use apps::fetcher::ContentFetcher;
 use endpoint::{Endpoint, Endpoints, EndpointPath};
 use handlers::{Redirection, extract_url, ContentHandler};
 use self::auth::{Authorization, Authorized};
@@ -45,7 +45,7 @@ pub struct Router<A: Authorization + 'static> {
 	control: Option<Control>,
 	main_page: &'static str,
 	endpoints: Arc<Endpoints>,
-	fetch: Arc<AppFetcher>,
+	fetch: Arc<ContentFetcher>,
 	special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
 	authorization: Arc<A>,
 	allowed_hosts: Option<Vec<String>>,
@@ -136,7 +136,7 @@ impl<A: Authorization> Router<A> {
 	pub fn new(
 		control: Control,
 		main_page: &'static str,
-		app_fetcher: Arc<AppFetcher>,
+		content_fetcher: Arc<ContentFetcher>,
 		endpoints: Arc<Endpoints>,
 		special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
 		authorization: Arc<A>,
@@ -148,7 +148,7 @@ impl<A: Authorization> Router<A> {
 			control: Some(control),
 			main_page: main_page,
 			endpoints: endpoints,
-			fetch: app_fetcher,
+			fetch: content_fetcher,
 			special: special,
 			authorization: authorization,
 			allowed_hosts: allowed_hosts,

dapps/src/tests/helpers.rs

@@ -17,7 +17,7 @@
 use std::env;
 use std::str;
 use std::sync::Arc;
-use rustc_serialize::hex::{ToHex, FromHex};
+use rustc_serialize::hex::FromHex;
 
 use ServerBuilder;
 use Server;

db/src/database.rs

@@ -460,7 +460,7 @@ mod client_tests {
 		crossbeam::scope(move |scope| {
 			let stop = Arc::new(AtomicBool::new(false));
 			run_worker(scope, stop.clone(), url);
-			let client = nanoipc::init_client::<DatabaseClient<_>>(url).unwrap();
+			let client = nanoipc::generic_client::<DatabaseClient<_>>(url).unwrap();
 			client.open_default(path.as_str().to_owned()).unwrap();
 			client.put("xxx".as_bytes(), "1".as_bytes()).unwrap();
 			client.close().unwrap();
@@ -477,7 +477,7 @@ mod client_tests {
 		crossbeam::scope(move |scope| {
 			let stop = Arc::new(AtomicBool::new(false));
 			run_worker(scope, stop.clone(), url);
-			let client = nanoipc::init_client::<DatabaseClient<_>>(url).unwrap();
+			let client = nanoipc::generic_client::<DatabaseClient<_>>(url).unwrap();
 
 			client.open_default(path.as_str().to_owned()).unwrap();
 			client.put("xxx".as_bytes(), "1".as_bytes()).unwrap();
@@ -498,7 +498,7 @@ mod client_tests {
 		crossbeam::scope(move |scope| {
 			let stop = Arc::new(AtomicBool::new(false));
 			run_worker(scope, stop.clone(), url);
-			let client = nanoipc::init_client::<DatabaseClient<_>>(url).unwrap();
+			let client = nanoipc::generic_client::<DatabaseClient<_>>(url).unwrap();
 
 			client.open_default(path.as_str().to_owned()).unwrap();
 			assert!(client.get("xxx".as_bytes()).unwrap().is_none());
@@ -516,7 +516,7 @@ mod client_tests {
 		crossbeam::scope(move |scope| {
 			let stop = Arc::new(AtomicBool::new(false));
 			run_worker(scope, stop.clone(), url);
-			let client = nanoipc::init_client::<DatabaseClient<_>>(url).unwrap();
+			let client = nanoipc::generic_client::<DatabaseClient<_>>(url).unwrap();
 			client.open_default(path.as_str().to_owned()).unwrap();
 
 			let transaction = DBTransaction::new();
@@ -541,7 +541,7 @@ mod client_tests {
 			let stop = StopGuard::new();
 			run_worker(&scope, stop.share(), url);
 
-			let client = nanoipc::init_client::<DatabaseClient<_>>(url).unwrap();
+			let client = nanoipc::generic_client::<DatabaseClient<_>>(url).unwrap();
 
 			client.open_default(path.as_str().to_owned()).unwrap();
 			let mut batch = Vec::new();

db/src/lib.rs.in

@@ -66,13 +66,13 @@ pub fn extras_service_url(db_path: &str) -> Result<String, ::std::io::Error> {
 
 pub fn blocks_client(db_path: &str) -> Result<DatabaseConnection, ServiceError> {
 	let url = try!(blocks_service_url(db_path));
-	let client = try!(nanoipc::init_client::<DatabaseClient<_>>(&url));
+	let client = try!(nanoipc::generic_client::<DatabaseClient<_>>(&url));
 	Ok(client)
 }
 
 pub fn extras_client(db_path: &str) -> Result<DatabaseConnection, ServiceError> {
 	let url = try!(extras_service_url(db_path));
-	let client = try!(nanoipc::init_client::<DatabaseClient<_>>(&url));
+	let client = try!(nanoipc::generic_client::<DatabaseClient<_>>(&url));
 	Ok(client)
 }
 

docker/ubuntu-aarch64/Dockerfile

@@ -23,15 +23,9 @@ RUN rustup target add aarch64-unknown-linux-gnu
 # show backtraces
 ENV RUST_BACKTRACE 1
 
-# set compilers
-ENV CXX aarch64-linux-gnu-g++
-ENV CC aarch64-linux-gnu-gcc
-
 # show tools
  RUN rustc -vV && \
- cargo -V && \
+ cargo -V 
- gcc -v &&\
- g++ -v
 
 # build parity
 RUN git clone https://github.com/ethcore/parity && \

docker/ubuntu-arm/Dockerfile

@@ -23,15 +23,10 @@ RUN rustup target add armv7-unknown-linux-gnueabihf
 # show backtraces
 ENV RUST_BACKTRACE 1
 
-# set compilers
-ENV CXX arm-linux-gnueabihf-g++
-ENV CC arm-linux-gnueabihf-gcc
 
 # show tools
  RUN rustc -vV && \
- cargo -V && \
+ cargo -V 
- gcc -v &&\
- g++ -v
 
 # build parity
 RUN git clone https://github.com/ethcore/parity && \

ethcore/Cargo.toml

@@ -51,3 +51,5 @@ dev = ["clippy"]
 default = []
 benches = []
 ipc = []
+ethkey-cli = ["ethkey/cli"]
+ethstore-cli = ["ethstore/cli"]

ethcore/build.rs

@@ -19,5 +19,6 @@ extern crate ethcore_ipc_codegen;
 fn main() {
 	ethcore_ipc_codegen::derive_binary("src/types/mod.rs.in").unwrap();
 	ethcore_ipc_codegen::derive_ipc("src/client/traits.rs").unwrap();
+	ethcore_ipc_codegen::derive_ipc("src/snapshot/snapshot_service_trait.rs").unwrap();
 	ethcore_ipc_codegen::derive_ipc("src/client/chain_notify.rs").unwrap();
 }

ethcore/src/block_queue.rs

@@ -37,7 +37,7 @@ const MIN_MEM_LIMIT: usize = 16384;
 const MIN_QUEUE_LIMIT: usize = 512;
 
 /// Block queue configuration
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct BlockQueueConfig {
 	/// Maximum number of blocks to keep in unverified queue.
 	/// When the limit is reached, is_full returns true.

ethcore/src/blockchain/config.rs

@@ -17,7 +17,7 @@
 //! Blockchain configuration.
 
 /// Blockchain configuration.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct Config {
 	/// Preferred cache size in bytes.
 	pub pref_cache_size: usize,

ethcore/src/client/chain_notify.rs

@@ -20,7 +20,7 @@ use util::H256;
 /// Represents what has to be handled by actor listening to chain events
 #[derive(Ipc)]
 pub trait ChainNotify : Send + Sync {
-	/// fires when chain has new blocks
+	/// fires when chain has new blocks.
 	fn new_blocks(&self,
 		_imported: Vec<H256>,
 		_invalid: Vec<H256>,

ethcore/src/client/client.rs

@@ -32,7 +32,7 @@ use util::kvdb::*;
 // other
 use io::*;
 use views::{BlockView, HeaderView, BodyView};
-use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult};
+use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError};
 use header::BlockNumber;
 use state::State;
 use spec::Spec;
@@ -122,11 +122,13 @@ impl SleepState {
 /// Call `import_block()` to import a block asynchronously; `flush_queue()` flushes the queue.
 pub struct Client {
 	mode: Mode,
-	chain: Arc<BlockChain>,
+	chain: RwLock<Arc<BlockChain>>,
-	tracedb: Arc<TraceDB<BlockChain>>,
+	tracedb: RwLock<TraceDB<BlockChain>>,
 	engine: Arc<Engine>,
-	db: Arc<Database>,
+	config: ClientConfig,
-	state_db: Mutex<Box<JournalDB>>,
+	db: RwLock<Arc<Database>>,
+	pruning: journaldb::Algorithm,
+	state_db: RwLock<Box<JournalDB>>,
 	block_queue: BlockQueue,
 	report: RwLock<ClientReport>,
 	import_lock: Mutex<()>,
@@ -159,17 +161,14 @@ impl Client {
 		path: &Path,
 		miner: Arc<Miner>,
 		message_channel: IoChannel<ClientIoMessage>,
+		db_config: &DatabaseConfig,
 	) -> Result<Arc<Client>, ClientError> {
 		let path = path.to_path_buf();
 		let gb = spec.genesis_block();
-		let mut db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
-		db_config.cache_size = config.db_cache_size;
-		db_config.compaction = config.db_compaction.compaction_profile();
-		db_config.wal = config.db_wal;
 
 		let db = Arc::new(try!(Database::open(&db_config, &path.to_str().unwrap()).map_err(ClientError::Database)));
-		let chain = Arc::new(BlockChain::new(config.blockchain, &gb, db.clone()));
+		let chain = Arc::new(BlockChain::new(config.blockchain.clone(), &gb, db.clone()));
-		let tracedb = Arc::new(try!(TraceDB::new(config.tracing, db.clone(), chain.clone())));
+		let tracedb = RwLock::new(try!(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone())));
 
 		let mut state_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE);
 		if state_db.is_empty() && try!(spec.ensure_db_good(state_db.as_hashdb_mut())) {
@@ -184,32 +183,34 @@ impl Client {
 
 		let engine = spec.engine.clone();
 
-		let block_queue = BlockQueue::new(config.queue, engine.clone(), message_channel.clone());
+		let block_queue = BlockQueue::new(config.queue.clone(), engine.clone(), message_channel.clone());
 		let panic_handler = PanicHandler::new_in_arc();
 		panic_handler.forward_from(&block_queue);
 
 		let awake = match config.mode { Mode::Dark(..) => false, _ => true };
 
 		let factories = Factories {
-			vm: EvmFactory::new(config.vm_type),
+			vm: EvmFactory::new(config.vm_type.clone()),
-			trie: TrieFactory::new(config.trie_spec),
+			trie: TrieFactory::new(config.trie_spec.clone()),
 			accountdb: Default::default(),
 		};
 
 		let client = Client {
 			sleep_state: Mutex::new(SleepState::new(awake)),
 			liveness: AtomicBool::new(awake),
-			mode: config.mode,
+			mode: config.mode.clone(),
-			chain: chain,
+			chain: RwLock::new(chain),
 			tracedb: tracedb,
 			engine: engine,
-			db: db,
+			pruning: config.pruning.clone(),
-			state_db: Mutex::new(state_db),
+			verifier: verification::new(config.verifier_type.clone()),
+			config: config,
+			db: RwLock::new(db),
+			state_db: RwLock::new(state_db),
 			block_queue: block_queue,
 			report: RwLock::new(Default::default()),
 			import_lock: Mutex::new(()),
 			panic_handler: panic_handler,
-			verifier: verification::new(config.verifier_type),
 			miner: miner,
 			io_channel: message_channel,
 			notify: RwLock::new(Vec::new()),
@@ -253,8 +254,9 @@ impl Client {
 		let mut last_hashes = LastHashes::new();
 		last_hashes.resize(256, H256::default());
 		last_hashes[0] = parent_hash;
+		let chain = self.chain.read();
 		for i in 0..255 {
-			match self.chain.block_details(&last_hashes[i]) {
+			match chain.block_details(&last_hashes[i]) {
 				Some(details) => {
 					last_hashes[i + 1] = details.parent.clone();
 				},
@@ -270,22 +272,23 @@ impl Client {
 		let engine = &*self.engine;
 		let header = &block.header;
 
+		let chain = self.chain.read();
 		// Check the block isn't so old we won't be able to enact it.
-		let best_block_number = self.chain.best_block_number();
+		let best_block_number = chain.best_block_number();
 		if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY {
 			warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number);
 			return Err(());
 		}
 
 		// Verify Block Family
-		let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &*self.chain);
+		let verify_family_result = self.verifier.verify_block_family(header, &block.bytes, engine, &**chain);
 		if let Err(e) = verify_family_result {
 			warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			return Err(());
 		};
 
 		// Check if Parent is in chain
-		let chain_has_parent = self.chain.block_header(header.parent_hash());
+		let chain_has_parent = chain.block_header(header.parent_hash());
 		if let None = chain_has_parent {
 			warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash());
 			return Err(());
@@ -294,9 +297,9 @@ impl Client {
 		// Enact Verified Block
 		let parent = chain_has_parent.unwrap();
 		let last_hashes = self.build_last_hashes(header.parent_hash().clone());
-		let db = self.state_db.lock().boxed_clone();
+		let db = self.state_db.read().boxed_clone();
 
-		let enact_result = enact_verified(block, engine, self.tracedb.tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
+		let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone());
 		if let Err(e) = enact_result {
 			warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e);
 			return Err(());
@@ -408,17 +411,18 @@ impl Client {
 			}
 		}
 
-		self.db.flush().expect("DB flush failed.");
+		self.db.read().flush().expect("DB flush failed.");
 		imported
 	}
 
 	fn commit_block<B>(&self, block: B, hash: &H256, block_data: &[u8]) -> ImportRoute where B: IsBlock + Drain {
 		let number = block.header().number();
 		let parent = block.header().parent_hash().clone();
+		let chain = self.chain.read();
 		// Are we committing an era?
 		let ancient = if number >= HISTORY {
 			let n = number - HISTORY;
-			Some((n, self.chain.block_hash(n).unwrap()))
+			Some((n, chain.block_hash(n).unwrap()))
 		} else {
 			None
 		};
@@ -432,14 +436,14 @@ impl Client {
 
 		//let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new));
 
-		let mut batch = DBTransaction::new(&self.db);
+		let mut batch = DBTransaction::new(&self.db.read());
 		// CHECK! I *think* this is fine, even if the state_root is equal to another
 		// already-imported block of the same number.
 		// TODO: Prove it with a test.
 		block.drain().commit(&mut batch, number, hash, ancient).expect("DB commit failed.");
 
-		let route = self.chain.insert_block(&mut batch, block_data, receipts);
+		let route = chain.insert_block(&mut batch, block_data, receipts);
-		self.tracedb.import(&mut batch, TraceImportRequest {
+		self.tracedb.read().import(&mut batch, TraceImportRequest {
 			traces: traces.into(),
 			block_hash: hash.clone(),
 			block_number: number,
@@ -447,8 +451,8 @@ impl Client {
 			retracted: route.retracted.len()
 		});
 		// Final commit to the DB
-		self.db.write_buffered(batch);
+		self.db.read().write_buffered(batch);
-		self.chain.commit();
+		chain.commit();
 
 		self.update_last_hashes(&parent, hash);
 		route
@@ -491,10 +495,10 @@ impl Client {
 		};
 
 		self.block_header(id).and_then(|header| {
-			let db = self.state_db.lock().boxed_clone();
+			let db = self.state_db.read().boxed_clone();
 
 			// early exit for pruned blocks
-			if db.is_pruned() && self.chain.best_block_number() >= block_number + HISTORY {
+			if db.is_pruned() && self.chain.read().best_block_number() >= block_number + HISTORY {
 				return None;
 			}
 
@@ -522,7 +526,7 @@ impl Client {
 	/// Get a copy of the best block's state.
 	pub fn state(&self) -> State {
 		State::from_existing(
-			self.state_db.lock().boxed_clone(),
+			self.state_db.read().boxed_clone(),
 			HeaderView::new(&self.best_block_header()).state_root(),
 			self.engine.account_start_nonce(),
 			self.factories.clone())
@@ -531,22 +535,22 @@ impl Client {
 
 	/// Get info on the cache.
 	pub fn blockchain_cache_info(&self) -> BlockChainCacheSize {
-		self.chain.cache_size()
+		self.chain.read().cache_size()
 	}
 
 	/// Get the report.
 	pub fn report(&self) -> ClientReport {
 		let mut report = self.report.read().clone();
-		report.state_db_mem = self.state_db.lock().mem_used();
+		report.state_db_mem = self.state_db.read().mem_used();
 		report
 	}
 
 	/// Tick the client.
 	// TODO: manage by real events.
 	pub fn tick(&self) {
-		self.chain.collect_garbage();
+		self.chain.read().collect_garbage();
 		self.block_queue.collect_garbage();
-		self.tracedb.collect_garbage();
+		self.tracedb.read().collect_garbage();
 
 		match self.mode {
 			Mode::Dark(timeout) => {
@@ -584,16 +588,16 @@ impl Client {
 	pub fn block_number(&self, id: BlockID) -> Option<BlockNumber> {
 		match id {
 			BlockID::Number(number) => Some(number),
-			BlockID::Hash(ref hash) => self.chain.block_number(hash),
+			BlockID::Hash(ref hash) => self.chain.read().block_number(hash),
 			BlockID::Earliest => Some(0),
-			BlockID::Latest | BlockID::Pending => Some(self.chain.best_block_number()),
+			BlockID::Latest | BlockID::Pending => Some(self.chain.read().best_block_number()),
 		}
 	}
 
 	/// Take a snapshot at the given block.
 	/// If the ID given is "latest", this will default to 1000 blocks behind.
-	pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), ::error::Error> {
+	pub fn take_snapshot<W: snapshot_io::SnapshotWriter + Send>(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), EthcoreError> {
-		let db = self.state_db.lock().boxed_clone();
+		let db = self.state_db.read().boxed_clone();
 		let best_block_number = self.chain_info().best_block_number;
 		let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at)));
 
@@ -618,7 +622,7 @@ impl Client {
 			},
 		};
 
-		try!(snapshot::take_snapshot(&self.chain, start_hash, db.as_hashdb(), writer, p));
+		try!(snapshot::take_snapshot(&self.chain.read(), start_hash, db.as_hashdb(), writer, p));
 
 		Ok(())
 	}
@@ -634,8 +638,8 @@ impl Client {
 
 	fn transaction_address(&self, id: TransactionID) -> Option<TransactionAddress> {
 		match id {
-			TransactionID::Hash(ref hash) => self.chain.transaction_address(hash),
+			TransactionID::Hash(ref hash) => self.chain.read().transaction_address(hash),
-			TransactionID::Location(id, index) => Self::block_hash(&self.chain, id).map(|hash| TransactionAddress {
+			TransactionID::Location(id, index) => Self::block_hash(&self.chain.read(), id).map(|hash| TransactionAddress {
 				block_hash: hash,
 				index: index,
 			})
@@ -666,6 +670,25 @@ impl Client {
 	}
 }
 
+impl snapshot::DatabaseRestore for Client {
+	/// Restart the client with a new backend
+	fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError> {
+		let _import_lock = self.import_lock.lock();
+		let mut state_db = self.state_db.write();
+		let mut chain = self.chain.write();
+		let mut tracedb = self.tracedb.write();
+		self.miner.clear();
+		let db = self.db.write();
+		try!(db.restore(new_db));
+
+		*state_db = journaldb::new(db.clone(), self.pruning, ::db::COL_STATE);
+		*chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone()));
+		*tracedb = try!(TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone()).map_err(ClientError::from));
+		Ok(())
+	}
+}
+
+
 impl BlockChainClient for Client {
 	fn call(&self, t: &SignedTransaction, block: BlockID, analytics: CallAnalytics) -> Result<Executed, CallError> {
 		let header = try!(self.block_header(block).ok_or(CallError::StatePruned));
@@ -749,15 +772,17 @@ impl BlockChainClient for Client {
 	}
 
 	fn best_block_header(&self) -> Bytes {
-		self.chain.best_block_header()
+		self.chain.read().best_block_header()
 	}
 
 	fn block_header(&self, id: BlockID) -> Option<Bytes> {
-		Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_header_data(&hash))
+		let chain = self.chain.read();
+		Self::block_hash(&chain, id).and_then(|hash| chain.block_header_data(&hash))
 	}
 
 	fn block_body(&self, id: BlockID) -> Option<Bytes> {
-		Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_body(&hash))
+		let chain = self.chain.read();
+		Self::block_hash(&chain, id).and_then(|hash| chain.block_body(&hash))
 	}
 
 	fn block(&self, id: BlockID) -> Option<Bytes> {
@@ -766,14 +791,16 @@ impl BlockChainClient for Client {
 				return Some(block.rlp_bytes(Seal::Without));
 			}
 		}
-		Self::block_hash(&self.chain, id).and_then(|hash| {
+		let chain = self.chain.read();
-			self.chain.block(&hash)
+		Self::block_hash(&chain, id).and_then(|hash| {
+			chain.block(&hash)
 		})
 	}
 
 	fn block_status(&self, id: BlockID) -> BlockStatus {
-		match Self::block_hash(&self.chain, id) {
+		let chain = self.chain.read();
-			Some(ref hash) if self.chain.is_known(hash) => BlockStatus::InChain,
+		match Self::block_hash(&chain, id) {
+			Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain,
 			Some(hash) => self.block_queue.block_status(&hash),
 			None => BlockStatus::Unknown
 		}
@@ -785,7 +812,8 @@ impl BlockChainClient for Client {
 				return Some(*block.header.difficulty() + self.block_total_difficulty(BlockID::Latest).expect("blocks in chain have details; qed"));
 			}
 		}
-		Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block_details(&hash)).map(|d| d.total_difficulty)
+		let chain = self.chain.read();
+		Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty)
 	}
 
 	fn nonce(&self, address: &Address, id: BlockID) -> Option<U256> {
@@ -793,7 +821,8 @@ impl BlockChainClient for Client {
 	}
 
 	fn block_hash(&self, id: BlockID) -> Option<H256> {
-		Self::block_hash(&self.chain, id)
+		let chain = self.chain.read();
+		Self::block_hash(&chain, id)
 	}
 
 	fn code(&self, address: &Address, id: BlockID) -> Option<Option<Bytes>> {
@@ -809,7 +838,7 @@ impl BlockChainClient for Client {
 	}
 
 	fn transaction(&self, id: TransactionID) -> Option<LocalizedTransaction> {
-		self.transaction_address(id).and_then(|address| self.chain.transaction(&address))
+		self.transaction_address(id).and_then(|address| self.chain.read().transaction(&address))
 	}
 
 	fn uncle(&self, id: UncleID) -> Option<Bytes> {
@@ -818,11 +847,12 @@ impl BlockChainClient for Client {
 	}
 
 	fn transaction_receipt(&self, id: TransactionID) -> Option<LocalizedReceipt> {
-		self.transaction_address(id).and_then(|address| self.chain.block_number(&address.block_hash).and_then(|block_number| {
+		let chain = self.chain.read();
-			let t = self.chain.block_body(&address.block_hash)
+		self.transaction_address(id).and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| {
+			let t = chain.block_body(&address.block_hash)
 				.and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index));
 
-			match (t, self.chain.transaction_receipt(&address)) {
+			match (t, chain.transaction_receipt(&address)) {
 				(Some(tx), Some(receipt)) => {
 					let block_hash = tx.block_hash.clone();
 					let block_number = tx.block_number.clone();
@@ -832,7 +862,7 @@ impl BlockChainClient for Client {
 						0 => U256::zero(),
 						i => {
 							let prior_address = TransactionAddress { block_hash: address.block_hash, index: i - 1 };
-							let prior_receipt = self.chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed");
+							let prior_receipt = chain.transaction_receipt(&prior_address).expect("Transaction receipt at `address` exists; `prior_address` has lower index in same block; qed");
 							prior_receipt.gas_used
 						}
 					};
@@ -863,28 +893,29 @@ impl BlockChainClient for Client {
 	}
 
 	fn tree_route(&self, from: &H256, to: &H256) -> Option<TreeRoute> {
-		match self.chain.is_known(from) && self.chain.is_known(to) {
+		let chain = self.chain.read();
-			true => Some(self.chain.tree_route(from.clone(), to.clone())),
+		match chain.is_known(from) && chain.is_known(to) {
+			true => Some(chain.tree_route(from.clone(), to.clone())),
 			false => None
 		}
 	}
 
 	fn find_uncles(&self, hash: &H256) -> Option<Vec<H256>> {
-		self.chain.find_uncle_hashes(hash, self.engine.maximum_uncle_age())
+		self.chain.read().find_uncle_hashes(hash, self.engine.maximum_uncle_age())
 	}
 
 	fn state_data(&self, hash: &H256) -> Option<Bytes> {
-		self.state_db.lock().state(hash)
+		self.state_db.read().state(hash)
 	}
 
 	fn block_receipts(&self, hash: &H256) -> Option<Bytes> {
-		self.chain.block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec())
+		self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts).to_vec())
 	}
 
 	fn import_block(&self, bytes: Bytes) -> Result<H256, BlockImportError> {
 		{
 			let header = BlockView::new(&bytes).header_view();
-			if self.chain.is_known(&header.sha3()) {
+			if self.chain.read().is_known(&header.sha3()) {
 				return Err(BlockImportError::Import(ImportError::AlreadyInChain));
 			}
 			if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown {
@@ -903,12 +934,13 @@ impl BlockChainClient for Client {
 	}
 
 	fn chain_info(&self) -> BlockChainInfo {
+		let chain = self.chain.read();
 		BlockChainInfo {
-			total_difficulty: self.chain.best_block_total_difficulty(),
+			total_difficulty: chain.best_block_total_difficulty(),
-			pending_total_difficulty: self.chain.best_block_total_difficulty(),
+			pending_total_difficulty: chain.best_block_total_difficulty(),
-			genesis_hash: self.chain.genesis_hash(),
+			genesis_hash: chain.genesis_hash(),
-			best_block_hash: self.chain.best_block_hash(),
+			best_block_hash: chain.best_block_hash(),
-			best_block_number: From::from(self.chain.best_block_number())
+			best_block_number: From::from(chain.best_block_number())
 		}
 	}
 
@@ -918,7 +950,7 @@ impl BlockChainClient for Client {
 
 	fn blocks_with_bloom(&self, bloom: &H2048, from_block: BlockID, to_block: BlockID) -> Option<Vec<BlockNumber>> {
 		match (self.block_number(from_block), self.block_number(to_block)) {
-			(Some(from), Some(to)) => Some(self.chain.blocks_with_bloom(bloom, from, to)),
+			(Some(from), Some(to)) => Some(self.chain.read().blocks_with_bloom(bloom, from, to)),
 			_ => None
 		}
 	}
@@ -936,10 +968,11 @@ impl BlockChainClient for Client {
 
 		blocks.sort();
 
+		let chain = self.chain.read();
 		blocks.into_iter()
-			.filter_map(|number| self.chain.block_hash(number).map(|hash| (number, hash)))
+			.filter_map(|number| chain.block_hash(number).map(|hash| (number, hash)))
-			.filter_map(|(number, hash)| self.chain.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
+			.filter_map(|(number, hash)| chain.block_receipts(&hash).map(|r| (number, hash, r.receipts)))
-			.filter_map(|(number, hash, receipts)| self.chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes())))
+			.filter_map(|(number, hash, receipts)| chain.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes())))
 			.flat_map(|(number, hash, receipts, hashes)| {
 				let mut log_index = 0;
 				receipts.into_iter()
@@ -975,7 +1008,7 @@ impl BlockChainClient for Client {
 				to_address: From::from(filter.to_address),
 			};
 
-			let traces = self.tracedb.filter(&filter);
+			let traces = self.tracedb.read().filter(&filter);
 			Some(traces)
 		} else {
 			None
@@ -987,7 +1020,7 @@ impl BlockChainClient for Client {
 		self.transaction_address(trace.transaction)
 			.and_then(|tx_address| {
 				self.block_number(BlockID::Hash(tx_address.block_hash))
-					.and_then(|number| self.tracedb.trace(number, tx_address.index, trace_address))
+					.and_then(|number| self.tracedb.read().trace(number, tx_address.index, trace_address))
 			})
 	}
 
@@ -995,17 +1028,17 @@ impl BlockChainClient for Client {
 		self.transaction_address(transaction)
 			.and_then(|tx_address| {
 				self.block_number(BlockID::Hash(tx_address.block_hash))
-					.and_then(|number| self.tracedb.transaction_traces(number, tx_address.index))
+					.and_then(|number| self.tracedb.read().transaction_traces(number, tx_address.index))
 			})
 	}
 
 	fn block_traces(&self, block: BlockID) -> Option<Vec<LocalizedTrace>> {
 		self.block_number(block)
-			.and_then(|number| self.tracedb.block_traces(number))
+			.and_then(|number| self.tracedb.read().block_traces(number))
 	}
 
 	fn last_hashes(&self) -> LastHashes {
-		(*self.build_last_hashes(self.chain.best_block_hash())).clone()
+		(*self.build_last_hashes(self.chain.read().best_block_hash())).clone()
 	}
 
 	fn queue_transactions(&self, transactions: Vec<Bytes>) {
@@ -1032,14 +1065,15 @@ impl BlockChainClient for Client {
 impl MiningBlockChainClient for Client {
 	fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock {
 		let engine = &*self.engine;
-		let h = self.chain.best_block_hash();
+		let chain = self.chain.read();
+		let h = chain.best_block_hash();
 
 		let mut open_block = OpenBlock::new(
 			engine,
 			self.factories.clone(),
 			false,	// TODO: this will need to be parameterised once we want to do immediate mining insertion.
-			self.state_db.lock().boxed_clone(),
+			self.state_db.read().boxed_clone(),
-			&self.chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
+			&chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"),
 			self.build_last_hashes(h.clone()),
 			author,
 			gas_range_target,
@@ -1047,7 +1081,7 @@ impl MiningBlockChainClient for Client {
 		).expect("OpenBlock::new only fails if parent state root invalid; state root of best block's header is never invalid; qed");
 
 		// Add uncles
-		self.chain
+		chain
 			.find_uncle_headers(&h, engine.maximum_uncle_age())
 			.unwrap()
 			.into_iter()
@@ -1088,7 +1122,7 @@ impl MiningBlockChainClient for Client {
 				precise_time_ns() - start,
 			);
 		});
-		self.db.flush().expect("DB flush failed.");
+		self.db.read().flush().expect("DB flush failed.");
 		Ok(h)
 	}
 }

ethcore/src/client/config.rs

@@ -62,7 +62,7 @@ impl FromStr for DatabaseCompactionProfile {
 }
 
 /// Operating mode for the client.
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Eq, PartialEq, Clone)]
 pub enum Mode {
 	/// Always on.
 	Active,

ethcore/src/engines/basic_authority.rs

@@ -187,7 +187,10 @@ mod tests {
 	use spec::Spec;
 
 	/// Create a new test chain spec with `BasicAuthority` consensus engine.
-	fn new_test_authority() -> Spec { Spec::load(include_bytes!("../../res/test_authority.json")) }
+	fn new_test_authority() -> Spec {
+		let bytes: &[u8] = include_bytes!("../../res/test_authority.json");
+		Spec::load(bytes).expect("invalid chain spec")
+	}
 
 	#[test]
 	fn has_valid_metadata() {

ethcore/src/engines/instant_seal.rs

@@ -72,7 +72,10 @@ mod tests {
 	use block::*;
 
 	/// Create a new test chain spec with `BasicAuthority` consensus engine.
-	fn new_test_instant() -> Spec { Spec::load(include_bytes!("../../res/instant_seal.json")) }
+	fn new_test_instant() -> Spec {
+		let bytes: &[u8] = include_bytes!("../../res/instant_seal.json");
+		Spec::load(bytes).expect("invalid chain spec")
+	}
 
 	#[test]
 	fn instant_can_seal() {

ethcore/src/ethereum/mod.rs

@@ -29,29 +29,33 @@ pub use self::denominations::*;
 
 use super::spec::*;
 
+fn load(b: &[u8]) -> Spec {
+	Spec::load(b).expect("chain spec is invalid")
+}
+
 /// Create a new Olympic chain spec.
-pub fn new_olympic() -> Spec { Spec::load(include_bytes!("../../res/ethereum/olympic.json")) }
+pub fn new_olympic() -> Spec { load(include_bytes!("../../res/ethereum/olympic.json")) }
 
 /// Create a new Frontier mainnet chain spec.
-pub fn new_frontier() -> Spec { Spec::load(include_bytes!("../../res/ethereum/frontier.json")) }
+pub fn new_frontier() -> Spec { load(include_bytes!("../../res/ethereum/frontier.json")) }
 
 /// Create a new Frontier mainnet chain spec without the DAO hardfork.
-pub fn new_classic() -> Spec { Spec::load(include_bytes!("../../res/ethereum/classic.json")) }
+pub fn new_classic() -> Spec { load(include_bytes!("../../res/ethereum/classic.json")) }
 
 /// Create a new Frontier chain spec as though it never changes to Homestead.
-pub fn new_frontier_test() -> Spec { Spec::load(include_bytes!("../../res/ethereum/frontier_test.json")) }
+pub fn new_frontier_test() -> Spec { load(include_bytes!("../../res/ethereum/frontier_test.json")) }
 
 /// Create a new Homestead chain spec as though it never changed from Frontier.
-pub fn new_homestead_test() -> Spec { Spec::load(include_bytes!("../../res/ethereum/homestead_test.json")) }
+pub fn new_homestead_test() -> Spec { load(include_bytes!("../../res/ethereum/homestead_test.json")) }
 
 /// Create a new Frontier/Homestead/DAO chain spec with transition points at #5 and #8.
-pub fn new_daohardfork_test() -> Spec { Spec::load(include_bytes!("../../res/ethereum/daohardfork_test.json")) }
+pub fn new_daohardfork_test() -> Spec { load(include_bytes!("../../res/ethereum/daohardfork_test.json")) }
 
 /// Create a new Frontier main net chain spec without genesis accounts.
-pub fn new_mainnet_like() -> Spec { Spec::load(include_bytes!("../../res/ethereum/frontier_like_test.json")) }
+pub fn new_mainnet_like() -> Spec { load(include_bytes!("../../res/ethereum/frontier_like_test.json")) }
 
 /// Create a new Morden chain spec.
-pub fn new_morden() -> Spec { Spec::load(include_bytes!("../../res/ethereum/morden.json")) }
+pub fn new_morden() -> Spec { load(include_bytes!("../../res/ethereum/morden.json")) }
 
 #[cfg(test)]
 mod tests {

ethcore/src/json_tests/chain.rs

@@ -58,12 +58,14 @@ pub fn json_chain_test(json_data: &[u8], era: ChainEra) -> Vec<String> {
 
 			let temp = RandomTempPath::new();
 			{
+				let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
 				let client = Client::new(
 					ClientConfig::default(),
 					&spec,
 					temp.as_path(),
 					Arc::new(Miner::with_spec(&spec)),
-					IoChannel::disconnected()
+					IoChannel::disconnected(),
+					&db_config,
 				).unwrap();
 				for b in &blockchain.blocks_rlp() {
 					if Block::is_good(&b) {

ethcore/src/miner/miner.rs

@@ -227,6 +227,11 @@ impl Miner {
 		self.options.force_sealing || !self.options.new_work_notify.is_empty()
 	}
 
+	/// Clear all pending block states
+	pub fn clear(&self) {
+		self.sealing_work.lock().queue.reset();
+	}
+
 	/// Get `Some` `clone()` of the current pending block's state or `None` if we're not sealing.
 	pub fn pending_state(&self) -> Option<State> {
 		self.sealing_work.lock().queue.peek_last_ref().map(|b| b.block().fields().state.clone())

ethcore/src/service.rs

@@ -23,7 +23,7 @@ use error::*;
 use client::{Client, ClientConfig, ChainNotify};
 use miner::Miner;
 use snapshot::ManifestData;
-use snapshot::service::Service as SnapshotService;
+use snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams};
 use std::sync::atomic::AtomicBool;
 
 #[cfg(feature="ipc")]
@@ -46,6 +46,8 @@ pub enum ClientIoMessage {
 	FeedStateChunk(H256, Bytes),
 	/// Feed a block chunk to the snapshot service
 	FeedBlockChunk(H256, Bytes),
+	/// Take a snapshot for the block with given number.
+	TakeSnapshot(u64),
 }
 
 /// Client service setup. Creates and registers client and network services with the IO subsystem.
@@ -58,11 +60,12 @@ pub struct ClientService {
 }
 
 impl ClientService {
-	/// Start the service in a separate thread.
+	/// Start the `ClientService`.
 	pub fn start(
 		config: ClientConfig,
 		spec: &Spec,
-		db_path: &Path,
+		client_path: &Path,
+		snapshot_path: &Path,
 		ipc_path: &Path,
 		miner: Arc<Miner>,
 		) -> Result<ClientService, Error>
@@ -76,11 +79,25 @@ impl ClientService {
 			warn!("Your chain is an alternative fork. {}", Colour::Red.bold().paint("TRANSACTIONS MAY BE REPLAYED ON THE MAINNET!"));
 		}
 
-		let pruning = config.pruning;
+		let mut db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
-		let client = try!(Client::new(config, &spec, db_path, miner, io_service.channel()));
+		db_config.cache_size = config.db_cache_size;
-		let snapshot = try!(SnapshotService::new(spec, pruning, db_path.into(), io_service.channel()));
+		db_config.compaction = config.db_compaction.compaction_profile();
+		db_config.wal = config.db_wal;
 
-		let snapshot = Arc::new(snapshot);
+		let pruning = config.pruning;
+		let client = try!(Client::new(config, &spec, client_path, miner, io_service.channel(), &db_config));
+
+		let snapshot_params = SnapServiceParams {
+			engine: spec.engine.clone(),
+			genesis_block: spec.genesis_block(),
+			db_config: db_config,
+			pruning: pruning,
+			channel: io_service.channel(),
+			snapshot_root: snapshot_path.into(),
+			client_db: client_path.into(),
+			db_restore: client.clone(),
+		};
+		let snapshot = Arc::new(try!(SnapshotService::new(snapshot_params)));
 
 		panic_handler.forward_from(&*client);
 		let client_io = Arc::new(ClientIoHandler {
@@ -90,7 +107,7 @@ impl ClientService {
 		try!(io_service.register_handler(client_io));
 
 		let stop_guard = ::devtools::StopGuard::new();
-		run_ipc(ipc_path, client.clone(), stop_guard.share());
+		run_ipc(ipc_path, client.clone(), snapshot.clone(), stop_guard.share());
 
 		Ok(ClientService {
 			io_service: Arc::new(io_service),
@@ -145,16 +162,22 @@ struct ClientIoHandler {
 }
 
 const CLIENT_TICK_TIMER: TimerToken = 0;
+const SNAPSHOT_TICK_TIMER: TimerToken = 1;
+
 const CLIENT_TICK_MS: u64 = 5000;
+const SNAPSHOT_TICK_MS: u64 = 10000;
 
 impl IoHandler<ClientIoMessage> for ClientIoHandler {
 	fn initialize(&self, io: &IoContext<ClientIoMessage>) {
 		io.register_timer(CLIENT_TICK_TIMER, CLIENT_TICK_MS).expect("Error registering client timer");
+		io.register_timer(SNAPSHOT_TICK_TIMER, SNAPSHOT_TICK_MS).expect("Error registering snapshot timer");
 	}
 
 	fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
-		if timer == CLIENT_TICK_TIMER {
+		match timer {
-			self.client.tick();
+			CLIENT_TICK_TIMER => self.client.tick(),
+			SNAPSHOT_TICK_TIMER => self.snapshot.tick(),
+			_ => warn!("IO service triggered unregistered timer '{}'", timer),
 		}
 	}
 
@@ -170,20 +193,38 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
 			}
 			ClientIoMessage::FeedStateChunk(ref hash, ref chunk) => self.snapshot.feed_state_chunk(*hash, chunk),
 			ClientIoMessage::FeedBlockChunk(ref hash, ref chunk) => self.snapshot.feed_block_chunk(*hash, chunk),
+			ClientIoMessage::TakeSnapshot(num) => {
+				if let Err(e) = self.snapshot.take_snapshot(&*self.client, num) {
+					warn!("Failed to take snapshot at block #{}: {}", num, e);
+				}
+			}
 			_ => {} // ignore other messages
 		}
 	}
 }
 
 #[cfg(feature="ipc")]
-fn run_ipc(base_path: &Path, client: Arc<Client>, stop: Arc<AtomicBool>) {
+fn run_ipc(base_path: &Path, client: Arc<Client>, snapshot_service: Arc<SnapshotService>, stop: Arc<AtomicBool>) {
 	let mut path = base_path.to_owned();
 	path.push("parity-chain.ipc");
 	let socket_addr = format!("ipc://{}", path.to_string_lossy());
+	let s = stop.clone();
 	::std::thread::spawn(move || {
 		let mut worker = nanoipc::Worker::new(&(client as Arc<BlockChainClient>));
 		worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues");
 
+		while !s.load(::std::sync::atomic::Ordering::Relaxed) {
+			worker.poll();
+		}
+	});
+
+	let mut path = base_path.to_owned();
+	path.push("parity-snapshot.ipc");
+	let socket_addr = format!("ipc://{}", path.to_string_lossy());
+	::std::thread::spawn(move || {
+		let mut worker = nanoipc::Worker::new(&(snapshot_service as Arc<::snapshot::SnapshotService>));
+		worker.add_reqrep(&socket_addr).expect("Ipc expected to initialize with no issues");
+
 		while !stop.load(::std::sync::atomic::Ordering::Relaxed) {
 			worker.poll();
 		}
@@ -191,7 +232,7 @@ fn run_ipc(base_path: &Path, client: Arc<Client>, stop: Arc<AtomicBool>) {
 }
 
 #[cfg(not(feature="ipc"))]
-fn run_ipc(_base_path: &Path, _client: Arc<Client>, _stop: Arc<AtomicBool>) {
+fn run_ipc(_base_path: &Path, _client: Arc<Client>, _snapshot_service: Arc<SnapshotService>, _stop: Arc<AtomicBool>) {
 }
 
 #[cfg(test)]
@@ -206,15 +247,25 @@ mod tests {
 	#[test]
 	fn it_can_be_started() {
 		let temp_path = RandomTempPath::new();
-		let mut path = temp_path.as_path().to_owned();
+		let path = temp_path.as_path().to_owned();
-		path.push("pruning");
+		let client_path = {
-		path.push("db");
+			let mut path = path.to_owned();
+			path.push("client");
+			path
+		};
+
+		let snapshot_path = {
+			let mut path = path.to_owned();
+			path.push("snapshot");
+			path
+		};
 
 		let spec = get_test_spec();
 		let service = ClientService::start(
 			ClientConfig::default(),
 			&spec,
-			&path,
+			&client_path,
+			&snapshot_path,
 			&path,
 			Arc::new(Miner::with_spec(&spec)),
 		);

ethcore/src/snapshot/mod.rs

@@ -32,9 +32,9 @@ use util::Mutex;
 use util::hash::{FixedHash, H256};
 use util::journaldb::{self, Algorithm, JournalDB};
 use util::kvdb::Database;
-use util::sha3::SHA3_NULL_RLP;
 use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
-use rlp::{DecoderError, RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
+use util::sha3::SHA3_NULL_RLP;
+use rlp::{RlpStream, Stream, UntrustedRlp, View, Compressible, RlpType};
 
 use self::account::Account;
 use self::block::AbridgedBlock;
@@ -44,7 +44,12 @@ use crossbeam::{scope, ScopedJoinHandle};
 use rand::{Rng, OsRng};
 
 pub use self::error::Error;
-pub use self::service::{RestorationStatus, Service, SnapshotService};
+
+pub use self::service::{Service, DatabaseRestore};
+pub use self::traits::{SnapshotService, RemoteSnapshotService};
+pub use self::watcher::Watcher;
+pub use types::snapshot_manifest::ManifestData;
+pub use types::restoration_status::RestorationStatus;
 
 pub mod io;
 pub mod service;
@@ -52,10 +57,16 @@ pub mod service;
 mod account;
 mod block;
 mod error;
+mod watcher;
 
 #[cfg(test)]
 mod tests;
 
+mod traits {
+	#![allow(dead_code, unused_assignments, unused_variables, missing_docs)] // codegen issues
+	include!(concat!(env!("OUT_DIR"), "/snapshot_service_trait.rs"));
+}
+
 // Try to have chunks be around 4MB (before compression)
 const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
 
@@ -72,17 +83,28 @@ pub struct Progress {
 }
 
 impl Progress {
+	/// Reset the progress.
+	pub fn reset(&self) {
+		self.accounts.store(0, Ordering::Release);
+		self.blocks.store(0, Ordering::Release);
+		self.size.store(0, Ordering::Release);
+
+		// atomic fence here to ensure the others are written first?
+		// logs might very rarely get polluted if not.
+		self.done.store(false, Ordering::Release);
+	}
+
 	/// Get the number of accounts snapshotted thus far.
-	pub fn accounts(&self) -> usize { self.accounts.load(Ordering::Relaxed) }
+	pub fn accounts(&self) -> usize { self.accounts.load(Ordering::Acquire) }
 
 	/// Get the number of blocks snapshotted thus far.
-	pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Relaxed) }
+	pub fn blocks(&self) -> usize { self.blocks.load(Ordering::Acquire) }
 
 	/// Get the written size of the snapshot in bytes.
-	pub fn size(&self) -> usize { self.size.load(Ordering::Relaxed) }
+	pub fn size(&self) -> usize { self.size.load(Ordering::Acquire) }
 
 	/// Whether the snapshot is complete.
-	pub fn done(&self) -> bool  { self.done.load(Ordering::SeqCst) }
+	pub fn done(&self) -> bool  { self.done.load(Ordering::Acquire) }
 
 }
 /// Take a snapshot using the given blockchain, starting block hash, and database, writing into the given writer.
@@ -354,54 +376,6 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex<SnapshotWriter +
 	Ok(chunker.hashes)
 }
 
-/// Manifest data.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ManifestData {
-	/// List of state chunk hashes.
-	pub state_hashes: Vec<H256>,
-	/// List of block chunk hashes.
-	pub block_hashes: Vec<H256>,
-	/// The final, expected state root.
-	pub state_root: H256,
-	/// Block number this snapshot was taken at.
-	pub block_number: u64,
-	/// Block hash this snapshot was taken at.
-	pub block_hash: H256,
-}
-
-impl ManifestData {
-	/// Encode the manifest data to rlp.
-	pub fn into_rlp(self) -> Bytes {
-		let mut stream = RlpStream::new_list(5);
-		stream.append(&self.state_hashes);
-		stream.append(&self.block_hashes);
-		stream.append(&self.state_root);
-		stream.append(&self.block_number);
-		stream.append(&self.block_hash);
-
-		stream.out()
-	}
-
-	/// Try to restore manifest data from raw bytes, interpreted as RLP.
-	pub fn from_rlp(raw: &[u8]) -> Result<Self, DecoderError> {
-		let decoder = UntrustedRlp::new(raw);
-
-		let state_hashes: Vec<H256> = try!(decoder.val_at(0));
-		let block_hashes: Vec<H256> = try!(decoder.val_at(1));
-		let state_root: H256 = try!(decoder.val_at(2));
-		let block_number: u64 = try!(decoder.val_at(3));
-		let block_hash: H256 = try!(decoder.val_at(4));
-
-		Ok(ManifestData {
-			state_hashes: state_hashes,
-			block_hashes: block_hashes,
-			state_root: state_root,
-			block_number: block_number,
-			block_hash: block_hash,
-		})
-	}
-}
-
 /// Used to rebuild the state trie piece by piece.
 pub struct StateRebuilder {
 	db: Box<JournalDB>,

ethcore/src/snapshot/service.rs

@@ -19,18 +19,19 @@
 use std::collections::HashSet;
 use std::io::ErrorKind;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::Arc;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
-use super::{ManifestData, StateRebuilder, BlockRebuilder};
+use super::{ManifestData, StateRebuilder, BlockRebuilder, RestorationStatus, SnapshotService};
 use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter};
 
 use blockchain::BlockChain;
+use client::Client;
 use engines::Engine;
 use error::Error;
+use ids::BlockID;
 use service::ClientIoMessage;
-use spec::Spec;
 
 use io::IoChannel;
 
@@ -39,51 +40,27 @@ use util::journaldb::Algorithm;
 use util::kvdb::{Database, DatabaseConfig};
 use util::snappy;
 
-/// Statuses for restorations.
+/// Helper for removing directories in case of error.
-#[derive(PartialEq, Eq, Clone, Copy, Debug)]
+struct Guard(bool, PathBuf);
-pub enum RestorationStatus {
+
-	///	No restoration.
+impl Guard {
-	Inactive,
+	fn new(path: PathBuf) -> Self { Guard(true, path) }
-	/// Ongoing restoration.
+
-	Ongoing,
+	fn disarm(mut self) { self.0 = false }
-	/// Failed restoration.
-	Failed,
 }
 
-/// The interface for a snapshot network service.
+impl Drop for Guard {
-/// This handles:
+	fn drop(&mut self) {
-///    - restoration of snapshots to temporary databases.
+		if self.0 {
-///    - responding to queries for snapshot manifests and chunks
+			let _ = fs::remove_dir_all(&self.1);
-pub trait SnapshotService {
+		}
-	/// Query the most recent manifest data.
+	}
-	fn manifest(&self) -> Option<ManifestData>;
+}
 
-	/// Get raw chunk for a given hash.
+/// External database restoration handler
-	fn chunk(&self, hash: H256) -> Option<Bytes>;
+pub trait DatabaseRestore: Send + Sync {
-
+	/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
-	/// Ask the snapshot service for the restoration status.
+	fn restore_db(&self, new_db: &str) -> Result<(), Error>;
-	fn status(&self) -> RestorationStatus;
-
-	/// Ask the snapshot service for the number of chunks completed.
-	/// Return a tuple of (state_chunks, block_chunks).
-	/// Undefined when not restoring.
-	fn chunks_done(&self) -> (usize, usize);
-
-	/// Begin snapshot restoration.
-	/// If restoration in-progress, this will reset it.
-	/// From this point on, any previous snapshot may become unavailable.
-	fn begin_restore(&self, manifest: ManifestData);
-
-	/// Abort an in-progress restoration if there is one.
-	fn abort_restore(&self);
-
-	/// Feed a raw state chunk to the service to be processed asynchronously.
-	/// no-op if not currently restoring.
-	fn restore_state_chunk(&self, hash: H256, chunk: Bytes);
-
-	/// Feed a raw block chunk to the service to be processed asynchronously.
-	/// no-op if currently restoring.
-	fn restore_block_chunk(&self, hash: H256, chunk: Bytes);
 }
 
 /// State restoration manager.
@@ -96,14 +73,17 @@ struct Restoration {
 	writer: LooseWriter,
 	snappy_buffer: Bytes,
 	final_state_root: H256,
+	guard: Guard,
 }
 
 struct RestorationParams<'a> {
 	manifest: ManifestData, // manifest to base restoration on.
 	pruning: Algorithm, // pruning algorithm for the database.
 	db_path: PathBuf, // database path
+	db_config: &'a DatabaseConfig,
 	writer: LooseWriter, // writer for recovered snapshot.
 	genesis: &'a [u8], // genesis block of the chain.
+	guard: Guard, // guard for the restoration directory.
 }
 
 impl Restoration {
@@ -114,8 +94,7 @@ impl Restoration {
 		let state_chunks = manifest.state_hashes.iter().cloned().collect();
 		let block_chunks = manifest.block_hashes.iter().cloned().collect();
 
-		let cfg = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+		let raw_db = Arc::new(try!(Database::open(params.db_config, &*params.db_path.to_string_lossy())
-		let raw_db = Arc::new(try!(Database::open(&cfg, &*params.db_path.to_string_lossy())
 			.map_err(UtilError::SimpleString)));
 
 		let chain = BlockChain::new(Default::default(), params.genesis, raw_db.clone());
@@ -131,6 +110,7 @@ impl Restoration {
 			writer: params.writer,
 			snappy_buffer: Vec::new(),
 			final_state_root: root,
+			guard: params.guard,
 		})
 	}
 
@@ -179,6 +159,7 @@ impl Restoration {
 
 		try!(self.writer.finish(self.manifest));
 
+		self.guard.disarm();
 		Ok(())
 	}
 
@@ -191,15 +172,35 @@ impl Restoration {
 /// Type alias for client io channel.
 pub type Channel = IoChannel<ClientIoMessage>;
 
-/// Service implementation.
+/// Snapshot service parameters.
-///
+pub struct ServiceParams {
-/// This will replace the client's state DB as soon as the last state chunk
+	/// The consensus engine this is built on.
-/// is fed, and will replace the client's blocks DB when the last block chunk
+	pub engine: Arc<Engine>,
-/// is fed.
+	/// The chain's genesis block.
+	pub genesis_block: Bytes,
+	/// Database configuration options.
+	pub db_config: DatabaseConfig,
+	/// State pruning algorithm.
+	pub pruning: Algorithm,
+	/// Async IO channel for sending messages.
+	pub channel: Channel,
+	/// The directory to put snapshots in.
+	/// Usually "<chain hash>/snapshot"
+	pub snapshot_root: PathBuf,
+	/// The client's database directory.
+	/// Usually "<chain hash>/<pruning>/db".
+	pub client_db: PathBuf,
+	/// A handle for database restoration.
+	pub db_restore: Arc<DatabaseRestore>,
+}
+
+/// `SnapshotService` implementation.
+/// This controls taking snapshots and restoring from them.
 pub struct Service {
 	restoration: Mutex<Option<Restoration>>,
-	client_db: PathBuf, // "<chain hash>/<pruning>/db"
+	client_db: PathBuf,
-	db_path: PathBuf,  // "<chain hash>/"
+	snapshot_root: PathBuf,
+	db_config: DatabaseConfig,
 	io_channel: Channel,
 	pruning: Algorithm,
 	status: Mutex<RestorationStatus>,
@@ -208,38 +209,34 @@ pub struct Service {
 	genesis_block: Bytes,
 	state_chunks: AtomicUsize,
 	block_chunks: AtomicUsize,
+	db_restore: Arc<DatabaseRestore>,
+	progress: super::Progress,
+	taking_snapshot: AtomicBool,
 }
 
 impl Service {
-	/// Create a new snapshot service.
+	/// Create a new snapshot service from the given parameters.
-	pub fn new(spec: &Spec, pruning: Algorithm, client_db: PathBuf, io_channel: Channel) -> Result<Self, Error> {
+	pub fn new(params: ServiceParams) -> Result<Self, Error> {
-		let db_path = try!(client_db.parent().and_then(Path::parent)
+		let mut service = Service {
-			.ok_or_else(|| UtilError::SimpleString("Failed to find database root.".into()))).to_owned();
-
-		let reader = {
-			let mut snapshot_path = db_path.clone();
-			snapshot_path.push("snapshot");
-			snapshot_path.push("current");
-
-			LooseReader::new(snapshot_path).ok()
-		};
-
-		let service = Service {
 			restoration: Mutex::new(None),
-			client_db: client_db,
+			client_db: params.client_db,
-			db_path: db_path,
+			snapshot_root: params.snapshot_root,
-			io_channel: io_channel,
+			db_config: params.db_config,
-			pruning: pruning,
+			io_channel: params.channel,
+			pruning: params.pruning,
 			status: Mutex::new(RestorationStatus::Inactive),
-			reader: RwLock::new(reader),
+			reader: RwLock::new(None),
-			engine: spec.engine.clone(),
+			engine: params.engine,
-			genesis_block: spec.genesis_block(),
+			genesis_block: params.genesis_block,
 			state_chunks: AtomicUsize::new(0),
 			block_chunks: AtomicUsize::new(0),
+			db_restore: params.db_restore,
+			progress: Default::default(),
+			taking_snapshot: AtomicBool::new(false),
 		};
 
 		// create the root snapshot dir if it doesn't exist.
-		if let Err(e) = fs::create_dir_all(service.root_dir()) {
+		if let Err(e) = fs::create_dir_all(&service.snapshot_root) {
 			if e.kind() != ErrorKind::AlreadyExists {
 				return Err(e.into())
 			}
@@ -252,26 +249,36 @@ impl Service {
 			}
 		}
 
-		Ok(service)
+		// delete the temporary snapshot dir if it does exist.
-	}
+		if let Err(e) = fs::remove_dir_all(service.temp_snapshot_dir()) {
+			if e.kind() != ErrorKind::NotFound {
+				return Err(e.into())
+			}
+		}
 
-	// get the root path.
+		let reader = LooseReader::new(service.snapshot_dir()).ok();
-	fn root_dir(&self) -> PathBuf {
+		*service.reader.get_mut() = reader;
-		let mut dir = self.db_path.clone();
+
-		dir.push("snapshot");
+		Ok(service)
-		dir
 	}
 
 	// get the current snapshot dir.
 	fn snapshot_dir(&self) -> PathBuf {
-		let mut dir = self.root_dir();
+		let mut dir = self.snapshot_root.clone();
 		dir.push("current");
 		dir
 	}
 
+	// get the temporary snapshot dir.
+	fn temp_snapshot_dir(&self) -> PathBuf {
+		let mut dir = self.snapshot_root.clone();
+		dir.push("in_progress");
+		dir
+	}
+
 	// get the restoration directory.
 	fn restoration_dir(&self) -> PathBuf {
-		let mut dir = self.root_dir();
+		let mut dir = self.snapshot_root.clone();
 		dir.push("restoration");
 		dir
 	}
@@ -295,37 +302,58 @@ impl Service {
 		let our_db = self.restoration_db();
 
 		trace!(target: "snapshot", "replacing {:?} with {:?}", self.client_db, our_db);
+		try!(self.db_restore.restore_db(our_db.to_str().unwrap()));
+		Ok(())
+	}
 
-		let mut backup_db = self.restoration_dir();
+	/// Tick the snapshot service. This will log any active snapshot
-		backup_db.push("backup_db");
+	/// being taken.
+	pub fn tick(&self) {
+		if self.progress.done() || !self.taking_snapshot.load(Ordering::SeqCst) { return }
 
-		let _ = fs::remove_dir_all(&backup_db);
+		let p = &self.progress;
+		info!("Snapshot: {} accounts {} blocks {} bytes", p.accounts(), p.blocks(), p.size());
+	}
 
-		let existed = match fs::rename(&self.client_db, &backup_db) {
+	/// Take a snapshot at the block with the given number.
-			Ok(_) => true,
+	/// calling this while a restoration is in progress or vice versa
-			Err(e) => if let ErrorKind::NotFound = e.kind() {
+	/// will lead to a race condition where the first one to finish will
-				false
+	/// have their produced snapshot overwritten.
-			} else {
+	pub fn take_snapshot(&self, client: &Client, num: u64) -> Result<(), Error> {
-				return Err(e.into());
+		if self.taking_snapshot.compare_and_swap(false, true, Ordering::SeqCst) {
-			}
+			info!("Skipping snapshot at #{} as another one is currently in-progress.", num);
-		};
+			return Ok(());
-
-		match fs::rename(&our_db, &self.client_db) {
-			Ok(_) => {
-				// clean up the backup.
-				if existed {
-					try!(fs::remove_dir_all(&backup_db));
-				}
-				Ok(())
-			}
-			Err(e) => {
-				// restore the backup.
-				if existed {
-					try!(fs::rename(&backup_db, &self.client_db));
-				}
-				Err(e.into())
-			}
 		}
+
+		info!("Taking snapshot at #{}", num);
+		self.progress.reset();
+
+		let temp_dir = self.temp_snapshot_dir();
+		let snapshot_dir = self.snapshot_dir();
+
+		let _ = fs::remove_dir_all(&temp_dir);
+
+		let writer = try!(LooseWriter::new(temp_dir.clone()));
+
+		let guard = Guard::new(temp_dir.clone());
+		let res = client.take_snapshot(writer, BlockID::Number(num), &self.progress);
+
+		self.taking_snapshot.store(false, Ordering::SeqCst);
+		try!(res);
+
+		info!("Finished taking snapshot at #{}", num);
+
+		let mut reader = self.reader.write();
+
+		// destroy the old snapshot reader.
+		*reader = None;
+
+		try!(fs::rename(temp_dir, &snapshot_dir));
+
+		*reader = Some(try!(LooseReader::new(snapshot_dir)));
+
+		guard.disarm();
+		Ok(())
 	}
 
 	/// Initialize the restoration synchronously.
@@ -354,13 +382,18 @@ impl Service {
 			manifest: manifest,
 			pruning: self.pruning,
 			db_path: self.restoration_db(),
+			db_config: &self.db_config,
 			writer: writer,
 			genesis: &self.genesis_block,
+			guard: Guard::new(rest_dir),
 		};
 
 		*res = Some(try!(Restoration::new(params)));
 
-		*self.status.lock() = RestorationStatus::Ongoing;
+		*self.status.lock() = RestorationStatus::Ongoing {
+			state_chunks_done: self.state_chunks.load(Ordering::Relaxed) as u32,
+			block_chunks_done: self.block_chunks.load(Ordering::Relaxed) as u32,
+		};
 		Ok(())
 	}
 
@@ -393,14 +426,7 @@ impl Service {
 		try!(fs::create_dir(&snapshot_dir));
 
 		trace!(target: "snapshot", "copying restored snapshot files over");
-		for maybe_file in try!(fs::read_dir(self.temp_recovery_dir())) {
+		try!(fs::rename(self.temp_recovery_dir(), &snapshot_dir));
-			let path = try!(maybe_file).path();
-			if let Some(name) = path.file_name().map(|x| x.to_owned()) {
-				let mut new_path = snapshot_dir.clone();
-				new_path.push(name);
-				try!(fs::rename(path, new_path));
-			}
-		}
 
 		let _ = fs::remove_dir_all(self.restoration_dir());
 
@@ -418,7 +444,7 @@ impl Service {
 
 		match self.status() {
 			RestorationStatus::Inactive | RestorationStatus::Failed => Ok(()),
-			RestorationStatus::Ongoing => {
+			RestorationStatus::Ongoing { .. } => {
 				let res = {
 					let rest = match *restoration {
 						Some(ref mut r) => r,
@@ -489,10 +515,6 @@ impl SnapshotService for Service {
 		*self.status.lock()
 	}
 
-	fn chunks_done(&self) -> (usize, usize) {
-		(self.state_chunks.load(Ordering::Relaxed), self.block_chunks.load(Ordering::Relaxed))
-	}
-
 	fn begin_restore(&self, manifest: ManifestData) {
 		self.io_channel.send(ClientIoMessage::BeginRestoration(manifest))
 			.expect("snapshot service and io service are kept alive by client service; qed");
@@ -520,37 +542,58 @@ impl SnapshotService for Service {
 	}
 }
 
+impl Drop for Service {
+	fn drop(&mut self) {
+		self.abort_restore();
+	}
+}
+
 #[cfg(test)]
 mod tests {
+	use std::sync::Arc;
 	use service::ClientIoMessage;
 	use io::{IoService};
 	use devtools::RandomTempPath;
 	use tests::helpers::get_test_spec;
 	use util::journaldb::Algorithm;
-
+	use error::Error;
-	use snapshot::ManifestData;
+	use snapshot::{ManifestData, RestorationStatus, SnapshotService};
 	use super::*;
 
+	struct NoopDBRestore;
+	impl DatabaseRestore for NoopDBRestore {
+		fn restore_db(&self, _new_db: &str) -> Result<(), Error> {
+			Ok(())
+		}
+	}
+
 	#[test]
 	fn sends_async_messages() {
 		let service = IoService::<ClientIoMessage>::start().unwrap();
+		let spec = get_test_spec();
 
 		let dir = RandomTempPath::new();
 		let mut dir = dir.as_path().to_owned();
-		dir.push("pruning");
+		let mut client_db = dir.clone();
-		dir.push("db");
+		dir.push("snapshot");
+		client_db.push("client");
 
-		let service = Service::new(
+		let snapshot_params = ServiceParams {
-			&get_test_spec(),
+			engine: spec.engine.clone(),
-			Algorithm::Archive,
+			genesis_block: spec.genesis_block(),
-			dir,
+			db_config: Default::default(),
-			service.channel()
+			pruning: Algorithm::Archive,
-		).unwrap();
+			channel: service.channel(),
+			snapshot_root: dir,
+			client_db: client_db,
+			db_restore: Arc::new(NoopDBRestore),
+		};
+
+		let service = Service::new(snapshot_params).unwrap();
 
 		assert!(service.manifest().is_none());
 		assert!(service.chunk(Default::default()).is_none());
 		assert_eq!(service.status(), RestorationStatus::Inactive);
-		assert_eq!(service.chunks_done(), (0, 0));
 
 		let manifest = ManifestData {
 			state_hashes: vec![],

ethcore/src/snapshot/snapshot_service_trait.rs

@@ -0,0 +1,54 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::{ManifestData, RestorationStatus};
+use util::{Bytes, H256};
+use ipc::IpcConfig;
+
+/// The interface for a snapshot network service.
+/// This handles:
+///    - restoration of snapshots to temporary databases.
+///    - responding to queries for snapshot manifests and chunks
+#[derive(Ipc)]
+#[ipc(client_ident="RemoteSnapshotService")]
+pub trait SnapshotService : Sync + Send {
+	/// Query the most recent manifest data.
+	fn manifest(&self) -> Option<ManifestData>;
+
+	/// Get raw chunk for a given hash.
+	fn chunk(&self, hash: H256) -> Option<Bytes>;
+
+	/// Ask the snapshot service for the restoration status.
+	fn status(&self) -> RestorationStatus;
+
+	/// Begin snapshot restoration.
+	/// If restoration in-progress, this will reset it.
+	/// From this point on, any previous snapshot may become unavailable.
+	fn begin_restore(&self, manifest: ManifestData);
+
+	/// Abort an in-progress restoration if there is one.
+	fn abort_restore(&self);
+
+	/// Feed a raw state chunk to the service to be processed asynchronously.
+	/// no-op if not currently restoring.
+	fn restore_state_chunk(&self, hash: H256, chunk: Bytes);
+
+	/// Feed a raw block chunk to the service to be processed asynchronously.
+	/// no-op if currently restoring.
+	fn restore_block_chunk(&self, hash: H256, chunk: Bytes);
+}
+
+impl IpcConfig for SnapshotService { }

ethcore/src/snapshot/watcher.rs

@@ -0,0 +1,203 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Watcher for snapshot-related chain events.
+
+use client::{BlockChainClient, Client, ChainNotify};
+use ids::BlockID;
+use service::ClientIoMessage;
+use views::HeaderView;
+
+use io::IoChannel;
+use util::hash::H256;
+
+use std::sync::Arc;
+
+// helper trait for transforming hashes to numbers and checking if syncing.
+trait Oracle: Send + Sync {
+	fn to_number(&self, hash: H256) -> Option<u64>;
+
+	fn is_major_syncing(&self) -> bool;
+}
+
+struct StandardOracle<F> where F: 'static + Send + Sync + Fn() -> bool {
+	client: Arc<Client>,
+	sync_status: F,
+}
+
+impl<F> Oracle for StandardOracle<F>
+	where F: Send + Sync + Fn() -> bool
+{
+	fn to_number(&self, hash: H256) -> Option<u64> {
+		self.client.block_header(BlockID::Hash(hash)).map(|h| HeaderView::new(&h).number())
+	}
+
+	fn is_major_syncing(&self) -> bool {
+		let queue_info = self.client.queue_info();
+
+		(self.sync_status)() || queue_info.unverified_queue_size + queue_info.verified_queue_size > 3
+	}
+}
+
+// helper trait for broadcasting a block to take a snapshot at.
+trait Broadcast: Send + Sync {
+	fn take_at(&self, num: Option<u64>);
+}
+
+impl Broadcast for IoChannel<ClientIoMessage> {
+	fn take_at(&self, num: Option<u64>) {
+		let num = match num {
+			Some(n) => n,
+			None => return,
+		};
+
+		trace!(target: "snapshot_watcher", "broadcast: {}", num);
+
+		if let Err(e) = self.send(ClientIoMessage::TakeSnapshot(num)) {
+			warn!("Snapshot watcher disconnected from IoService: {}", e);
+		}
+	}
+}
+
+/// A `ChainNotify` implementation which will trigger a snapshot event
+/// at certain block numbers.
+pub struct Watcher {
+	oracle: Box<Oracle>,
+	broadcast: Box<Broadcast>,
+	period: u64,
+	history: u64,
+}
+
+impl Watcher {
+	/// Create a new `Watcher` which will trigger a snapshot event
+	/// once every `period` blocks, but only after that block is
+	/// `history` blocks old.
+	pub fn new<F>(client: Arc<Client>, sync_status: F, channel: IoChannel<ClientIoMessage>, period: u64, history: u64) -> Self
+		where F: 'static + Send + Sync + Fn() -> bool
+	{
+		Watcher {
+			oracle: Box::new(StandardOracle {
+				client: client,
+				sync_status: sync_status,
+			}),
+			broadcast: Box::new(channel),
+			period: period,
+			history: history,
+		}
+	}
+}
+
+impl ChainNotify for Watcher {
+	fn new_blocks(
+		&self,
+		imported: Vec<H256>,
+		_: Vec<H256>,
+		_: Vec<H256>,
+		_: Vec<H256>,
+		_: Vec<H256>,
+		_duration: u64)
+	{
+		if self.oracle.is_major_syncing() { return }
+
+		trace!(target: "snapshot_watcher", "{} imported", imported.len());
+
+		let highest = imported.into_iter()
+			.filter_map(|h| self.oracle.to_number(h))
+			.filter(|&num| num >= self.period + self.history)
+			.map(|num| num - self.history)
+			.filter(|num| num % self.period == 0)
+			.fold(0, ::std::cmp::max);
+
+		match highest {
+			0 => self.broadcast.take_at(None),
+			_ => self.broadcast.take_at(Some(highest)),
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::{Broadcast, Oracle, Watcher};
+
+	use client::ChainNotify;
+
+	use util::{H256, U256};
+
+	use std::collections::HashMap;
+
+	struct TestOracle(HashMap<H256, u64>);
+
+	impl Oracle for TestOracle {
+		fn to_number(&self, hash: H256) -> Option<u64> {
+			self.0.get(&hash).cloned()
+		}
+
+		fn is_major_syncing(&self) -> bool { false }
+	}
+
+	struct TestBroadcast(Option<u64>);
+	impl Broadcast for TestBroadcast {
+		fn take_at(&self, num: Option<u64>) {
+			if num != self.0 {
+				panic!("Watcher broadcast wrong number. Expected {:?}, found {:?}", self.0, num);
+			}
+		}
+	}
+
+	// helper harness for tests which expect a notification.
+	fn harness(numbers: Vec<u64>, period: u64, history: u64, expected: Option<u64>) {
+		let hashes: Vec<_> = numbers.clone().into_iter().map(|x| H256::from(U256::from(x))).collect();
+		let map = hashes.clone().into_iter().zip(numbers).collect();
+
+		let watcher = Watcher {
+			oracle: Box::new(TestOracle(map)),
+			broadcast: Box::new(TestBroadcast(expected)),
+			period: period,
+			history: history,
+		};
+
+		watcher.new_blocks(
+			hashes,
+			vec![],
+			vec![],
+			vec![],
+			vec![],
+			0,
+		);
+	}
+
+	// helper
+
+	#[test]
+	fn should_not_fire() {
+		harness(vec![0], 5, 0, None);
+	}
+
+	#[test]
+	fn fires_once_for_two() {
+		harness(vec![14, 15], 10, 5, Some(10));
+	}
+
+	#[test]
+	fn finds_highest() {
+		harness(vec![15, 25], 10, 5, Some(20));
+	}
+
+	#[test]
+	fn doesnt_fire_before_history() {
+		harness(vec![10, 11], 10, 5, None);
+	}
+}

ethcore/src/spec/spec.rs

@@ -244,18 +244,21 @@ impl Spec {
 	}
 
 	/// Loads spec from json file.
-	pub fn load(reader: &[u8]) -> Self {
+	pub fn load<R>(reader: R) -> Result<Self, String> where R: Read {
-		From::from(ethjson::spec::Spec::load(reader).expect("invalid json file"))
+		match ethjson::spec::Spec::load(reader) {
+			Ok(spec) => Ok(spec.into()),
+			_ => Err("Spec json is invalid".into()),
+		}
 	}
 
 	/// Create a new Spec which conforms to the Frontier-era Morden chain except that it's a NullEngine consensus.
-	pub fn new_test() -> Spec {
+	pub fn new_test() -> Self {
-		Spec::load(include_bytes!("../../res/null_morden.json"))
+		Spec::load(include_bytes!("../../res/null_morden.json") as &[u8]).expect("null_morden.json is invalid")
 	}
 
 	/// Create a new Spec which is a NullEngine consensus with a premine of address whose secret is sha3('').
-	pub fn new_null() -> Spec {
+	pub fn new_null() -> Self {
-		Spec::load(include_bytes!("../../res/null.json"))
+		Spec::load(include_bytes!("../../res/null.json") as &[u8]).expect("null.json is invalid")
 	}
 }
 
@@ -267,6 +270,12 @@ mod tests {
 	use views::*;
 	use super::*;
 
+	// https://github.com/ethcore/parity/issues/1840
+	#[test]
+	fn test_load_empty() {
+		assert!(Spec::load(&vec![] as &[u8]).is_err());
+	}
+
 	#[test]
 	fn test_chain() {
 		let test_spec = Spec::new_test();

ethcore/src/tests/client.rs

@@ -28,7 +28,16 @@ use rlp::{Rlp, View};
 fn imports_from_empty() {
 	let dir = RandomTempPath::new();
 	let spec = get_test_spec();
-	let client = Client::new(ClientConfig::default(), &spec, dir.as_path(), Arc::new(Miner::with_spec(&spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
+	let client = Client::new(
+		ClientConfig::default(),
+		&spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
 	client.import_verified_blocks();
 	client.flush_queue();
 }
@@ -37,7 +46,16 @@ fn imports_from_empty() {
 fn should_return_registrar() {
 	let dir = RandomTempPath::new();
 	let spec = ethereum::new_morden();
-	let client = Client::new(ClientConfig::default(), &spec, dir.as_path(), Arc::new(Miner::with_spec(&spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
+	let client = Client::new(
+		ClientConfig::default(),
+		&spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
 	assert_eq!(client.additional_params().get("registrar"), Some(&"8e4e9b13d4b45cb0befc93c3061b1408f67316b2".to_owned()));
 }
 
@@ -55,7 +73,16 @@ fn returns_state_root_basic() {
 fn imports_good_block() {
 	let dir = RandomTempPath::new();
 	let spec = get_test_spec();
-	let client = Client::new(ClientConfig::default(), &spec, dir.as_path(), Arc::new(Miner::with_spec(&spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
+	let client = Client::new(
+		ClientConfig::default(),
+		&spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
 	let good_block = get_good_dummy_block();
 	if let Err(_) = client.import_block(good_block) {
 		panic!("error importing block being good by definition");
@@ -71,8 +98,16 @@ fn imports_good_block() {
 fn query_none_block() {
 	let dir = RandomTempPath::new();
 	let spec = get_test_spec();
-	let client = Client::new(ClientConfig::default(), &spec, dir.as_path(), Arc::new(Miner::with_spec(&spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
 
+	let client = Client::new(
+		ClientConfig::default(),
+		&spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
     let non_existant = client.block_header(BlockID::Number(188));
 	assert!(non_existant.is_none());
 }

ethcore/src/tests/helpers.rs

@@ -133,9 +133,17 @@ pub fn generate_dummy_client_with_data(block_number: u32, txs_per_block: usize,
 
 pub fn generate_dummy_client_with_spec_and_data<F>(get_test_spec: F, block_number: u32, txs_per_block: usize, tx_gas_prices: &[U256]) -> GuardedTempResult<Arc<Client>> where F: Fn()->Spec {
 	let dir = RandomTempPath::new();
-
 	let test_spec = get_test_spec();
-	let client = Client::new(ClientConfig::default(), &test_spec, dir.as_path(), Arc::new(Miner::with_spec(&test_spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
+	let client = Client::new(
+		ClientConfig::default(),
+		&test_spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&test_spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
 	let test_engine = &*test_spec.engine;
 
 	let mut db_result = get_temp_journal_db();
@@ -233,7 +241,17 @@ pub fn push_blocks_to_client(client: &Arc<Client>, timestamp_salt: u64, starting
 pub fn get_test_client_with_blocks(blocks: Vec<Bytes>) -> GuardedTempResult<Arc<Client>> {
 	let dir = RandomTempPath::new();
 	let test_spec = get_test_spec();
-	let client = Client::new(ClientConfig::default(), &test_spec, dir.as_path(), Arc::new(Miner::with_spec(&test_spec)), IoChannel::disconnected()).unwrap();
+	let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
+	let client = Client::new(
+		ClientConfig::default(),
+		&test_spec,
+		dir.as_path(),
+		Arc::new(Miner::with_spec(&test_spec)),
+		IoChannel::disconnected(),
+		&db_config
+	).unwrap();
+
 	for block in &blocks {
 		if let Err(_) = client.import_block(block.clone()) {
 			panic!("panic importing block which is well-formed");

ethcore/src/tests/rpc.rs

@@ -25,18 +25,23 @@ use devtools::*;
 use miner::Miner;
 use crossbeam;
 use io::IoChannel;
+use util::kvdb::DatabaseConfig;
 
 pub fn run_test_worker(scope: &crossbeam::Scope, stop: Arc<AtomicBool>, socket_path: &str) {
 	let socket_path = socket_path.to_owned();
 	scope.spawn(move || {
 		let temp = RandomTempPath::create_dir();
 		let spec = get_test_spec();
+		let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS);
+
 		let client = Client::new(
 			ClientConfig::default(),
 			&spec,
 			temp.as_path(),
 			Arc::new(Miner::with_spec(&spec)),
-			IoChannel::disconnected()).unwrap();
+			IoChannel::disconnected(),
+			&db_config
+		).unwrap();
 		let mut worker = nanoipc::Worker::new(&(client as Arc<BlockChainClient>));
 		worker.add_reqrep(&socket_path).unwrap();
 		while !stop.load(Ordering::Relaxed) {
@@ -51,7 +56,7 @@ fn can_handshake() {
 		let stop_guard = StopGuard::new();
 		let socket_path = "ipc:///tmp/parity-client-rpc-10.ipc";
 		run_test_worker(scope, stop_guard.share(), socket_path);
-		let remote_client = nanoipc::init_client::<RemoteClient<_>>(socket_path).unwrap();
+		let remote_client = nanoipc::generic_client::<RemoteClient<_>>(socket_path).unwrap();
 
 		assert!(remote_client.handshake().is_ok());
 	})
@@ -63,7 +68,7 @@ fn can_query_block() {
 		let stop_guard = StopGuard::new();
 		let socket_path = "ipc:///tmp/parity-client-rpc-20.ipc";
 		run_test_worker(scope, stop_guard.share(), socket_path);
-		let remote_client = nanoipc::init_client::<RemoteClient<_>>(socket_path).unwrap();
+		let remote_client = nanoipc::generic_client::<RemoteClient<_>>(socket_path).unwrap();
 
 		let non_existant_block = remote_client.block_header(BlockID::Number(999));
 

ethcore/src/types/mod.rs.in

@@ -31,3 +31,5 @@ pub mod trace_filter;
 pub mod call_analytics;
 pub mod transaction_import;
 pub mod block_import_error;
+pub mod restoration_status;
+pub mod snapshot_manifest;

ethcore/src/types/restoration_status.rs

@@ -0,0 +1,34 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Restoration status type definition
+
+/// Statuses for restorations.
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Binary)]
+pub enum RestorationStatus {
+	///	No restoration.
+	Inactive,
+	/// Ongoing restoration.
+	Ongoing {
+		/// Number of state chunks completed.
+		state_chunks_done: u32,
+		/// Number of block chunks completed.
+		block_chunks_done: u32,
+	},
+	/// Failed restoration.
+	Failed,
+}
+

ethcore/src/types/snapshot_manifest.rs

@@ -0,0 +1,70 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Snapshot manifest type definition
+
+use util::hash::H256;
+use rlp::*;
+use util::Bytes;
+
+/// Manifest data.
+#[derive(Debug, Clone, PartialEq, Eq, Binary)]
+pub struct ManifestData {
+	/// List of state chunk hashes.
+	pub state_hashes: Vec<H256>,
+	/// List of block chunk hashes.
+	pub block_hashes: Vec<H256>,
+	/// The final, expected state root.
+	pub state_root: H256,
+	/// Block number this snapshot was taken at.
+	pub block_number: u64,
+	/// Block hash this snapshot was taken at.
+	pub block_hash: H256,
+}
+
+impl ManifestData {
+	/// Encode the manifest data to rlp.
+	pub fn into_rlp(self) -> Bytes {
+		let mut stream = RlpStream::new_list(5);
+		stream.append(&self.state_hashes);
+		stream.append(&self.block_hashes);
+		stream.append(&self.state_root);
+		stream.append(&self.block_number);
+		stream.append(&self.block_hash);
+
+		stream.out()
+	}
+
+	/// Try to restore manifest data from raw bytes, interpreted as RLP.
+	pub fn from_rlp(raw: &[u8]) -> Result<Self, DecoderError> {
+		let decoder = UntrustedRlp::new(raw);
+
+		let state_hashes: Vec<H256> = try!(decoder.val_at(0));
+		let block_hashes: Vec<H256> = try!(decoder.val_at(1));
+		let state_root: H256 = try!(decoder.val_at(2));
+		let block_number: u64 = try!(decoder.val_at(3));
+		let block_hash: H256 = try!(decoder.val_at(4));
+
+		Ok(ManifestData {
+			state_hashes: state_hashes,
+			block_hashes: block_hashes,
+			state_root: state_root,
+			block_number: block_number,
+			block_hash: block_hash,
+		})
+	}
+}
+

ethcore/src/verification/mod.rs

@@ -25,7 +25,7 @@ pub use self::canon_verifier::CanonVerifier;
 pub use self::noop_verifier::NoopVerifier;
 
 /// Verifier type.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum VerifierType {
 	/// Verifies block normally.
 	Canon,

ipc/hypervisor/src/lib.rs

@@ -240,7 +240,7 @@ mod tests {
 		::std::thread::spawn(move || {
 			while !hypervisor_ready.load(Ordering::Relaxed) { }
 
-			let client = nanoipc::init_client::<HypervisorServiceClient<_>>(url).unwrap();
+			let client = nanoipc::fast_client::<HypervisorServiceClient<_>>(url).unwrap();
 			client.handshake().unwrap();
 			client.module_ready(test_module_id);
 		});

ipc/hypervisor/src/service.rs.in

@@ -110,7 +110,7 @@ impl HypervisorService {
 		let modules = self.modules.read().unwrap();
 		modules.get(&module_id).map(|module| {
 			trace!(target: "hypervisor", "Sending shutdown to {}({})", module_id, &module.control_url);
-			let client = nanoipc::init_client::<ControlServiceClient<_>>(&module.control_url).unwrap();
+			let client = nanoipc::fast_client::<ControlServiceClient<_>>(&module.control_url).unwrap();
 			client.shutdown();
 			trace!(target: "hypervisor", "Sent shutdown to {}", module_id);
 		});

ipc/nano/Cargo.toml

@@ -10,4 +10,4 @@ license = "GPL-3.0"
 ethcore-ipc = { path = "../rpc" }
 nanomsg = { git = "https://github.com/ethcore/nanomsg.rs.git" }
 log = "0.3"
-
+lazy_static = "0.2"

ipc/nano/src/lib.rs

@@ -19,6 +19,7 @@
 extern crate ethcore_ipc as ipc;
 extern crate nanomsg;
 #[macro_use] extern crate log;
+#[macro_use] extern crate lazy_static;
 
 pub use ipc::{WithSocket, IpcInterface, IpcConfig};
 pub use nanomsg::Socket as NanoSocket;
@@ -28,7 +29,8 @@ use nanomsg::{Socket, Protocol, Error, Endpoint, PollRequest, PollFd, PollInOut}
 use std::ops::Deref;
 
 const POLL_TIMEOUT: isize = 200;
-const CLIENT_CONNECTION_TIMEOUT: isize = 120000;
+const DEFAULT_CONNECTION_TIMEOUT: isize = 30000;
+const DEBUG_CONNECTION_TIMEOUT: isize = 5000;
 
 /// Generic worker to handle service (binded) sockets
 pub struct Worker<S: ?Sized> where S: IpcInterface {
@@ -68,7 +70,7 @@ pub fn init_duplex_client<S>(socket_addr: &str) -> Result<GuardedSocket<S>, Sock
 		SocketError::DuplexLink
 	}));
 
-	socket.set_receive_timeout(CLIENT_CONNECTION_TIMEOUT).unwrap();
+	socket.set_receive_timeout(DEFAULT_CONNECTION_TIMEOUT).unwrap();
 
 	let endpoint = try!(socket.connect(socket_addr).map_err(|e| {
 		warn!(target: "ipc", "Failed to bind socket to address '{}': {:?}", socket_addr, e);
@@ -84,26 +86,58 @@ pub fn init_duplex_client<S>(socket_addr: &str) -> Result<GuardedSocket<S>, Sock
 /// Spawns client <`S`> over specified address
 /// creates socket and connects endpoint to it
 /// for request-reply connections to the service
-pub fn init_client<S>(socket_addr: &str) -> Result<GuardedSocket<S>, SocketError> where S: WithSocket<Socket> {
+pub fn client<S>(socket_addr: &str, receive_timeout: Option<isize>) -> Result<GuardedSocket<S>, SocketError> where S: WithSocket<Socket> {
 	let mut socket = try!(Socket::new(Protocol::Req).map_err(|e| {
 		warn!(target: "ipc", "Failed to create ipc socket: {:?}", e);
 		SocketError::RequestLink
 	}));
 
-	socket.set_receive_timeout(CLIENT_CONNECTION_TIMEOUT).unwrap();
+	if let Some(timeout) = receive_timeout {
+		socket.set_receive_timeout(timeout).unwrap();
+	}
 
 	let endpoint = try!(socket.connect(socket_addr).map_err(|e| {
 		warn!(target: "ipc", "Failed to bind socket to address '{}': {:?}", socket_addr, e);
 		SocketError::RequestLink
 	}));
 
-	trace!(target: "ipc", "Created cleint for {}", socket_addr);
+	trace!(target: "ipc", "Created client for {}", socket_addr);
 	Ok(GuardedSocket {
 		client: Arc::new(S::init(socket)),
 		_endpoint: endpoint,
 	})
 }
 
+lazy_static! {
+	/// Set PARITY_IPC_DEBUG=1 for fail-fast connectivity problems diagnostic
+	pub static ref DEBUG_FLAG: bool = {
+		use std::env;
+
+		if let Ok(debug) = env::var("PARITY_IPC_DEBUG") {
+			debug == "1" || debug.to_uppercase() == "TRUE"
+		}
+		else { false }
+	};
+}
+
+/// Client with no default timeout on operations
+pub fn generic_client<S>(socket_addr: &str) -> Result<GuardedSocket<S>, SocketError> where S: WithSocket<Socket> {
+	if *DEBUG_FLAG {
+		client(socket_addr, Some(DEBUG_CONNECTION_TIMEOUT))
+	} else {
+		client(socket_addr, None)
+	}
+}
+
+/// Client over interface that is supposed to give quick almost non-blocking responses
+pub fn fast_client<S>(socket_addr: &str) -> Result<GuardedSocket<S>, SocketError> where S: WithSocket<Socket> {
+	if *DEBUG_FLAG {
+		client(socket_addr, Some(DEBUG_CONNECTION_TIMEOUT))
+	} else {
+		client(socket_addr, Some(DEFAULT_CONNECTION_TIMEOUT))
+	}
+}
+
 /// Error occurred while establising socket or endpoint
 #[derive(Debug)]
 pub enum SocketError {

parity/blockchain.rs

@@ -19,7 +19,6 @@ use std::{io, fs};
 use std::io::{BufReader, BufRead};
 use std::time::Duration;
 use std::thread::sleep;
-use std::path::Path;
 use std::sync::Arc;
 use rustc_serialize::hex::FromHex;
 use ethcore_logger::{setup_log, Config as LogConfig};
@@ -125,8 +124,9 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
 	// select pruning algorithm
 	let algorithm = cmd.pruning.to_algorithm(&cmd.dirs, genesis_hash, spec.fork_name.as_ref());
 
-	// prepare client_path
+	// prepare client and snapshot paths.
 	let client_path = cmd.dirs.client_path(genesis_hash, spec.fork_name.as_ref(), algorithm);
+	let snapshot_path = cmd.dirs.snapshot_path(genesis_hash, spec.fork_name.as_ref());
 
 	// execute upgrades
 	try!(execute_upgrades(&cmd.dirs, genesis_hash, spec.fork_name.as_ref(), algorithm, cmd.compaction.compaction_profile()));
@@ -138,8 +138,9 @@ fn execute_import(cmd: ImportBlockchain) -> Result<String, String> {
 	let service = try!(ClientService::start(
 		client_config,
 		&spec,
-		Path::new(&client_path),
+		&client_path,
-		Path::new(&cmd.dirs.ipc_path()),
+		&snapshot_path,
+		&cmd.dirs.ipc_path(),
 		Arc::new(Miner::with_spec(&spec)),
 	).map_err(|e| format!("Client service error: {:?}", e)));
 
@@ -237,8 +238,9 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
 	// select pruning algorithm
 	let algorithm = cmd.pruning.to_algorithm(&cmd.dirs, genesis_hash, spec.fork_name.as_ref());
 
-	// prepare client_path
+	// prepare client and snapshot paths.
 	let client_path = cmd.dirs.client_path(genesis_hash, spec.fork_name.as_ref(), algorithm);
+	let snapshot_path = cmd.dirs.snapshot_path(genesis_hash, spec.fork_name.as_ref());
 
 	// execute upgrades
 	try!(execute_upgrades(&cmd.dirs, genesis_hash, spec.fork_name.as_ref(), algorithm, cmd.compaction.compaction_profile()));
@@ -249,8 +251,9 @@ fn execute_export(cmd: ExportBlockchain) -> Result<String, String> {
 	let service = try!(ClientService::start(
 		client_config,
 		&spec,
-		Path::new(&client_path),
+		&client_path,
-		Path::new(&cmd.dirs.ipc_path()),
+		&snapshot_path,
+		&cmd.dirs.ipc_path(),
 		Arc::new(Miner::with_spec(&spec)),
 	).map_err(|e| format!("Client service error: {:?}", e)));
 

parity/boot.rs

@@ -63,7 +63,7 @@ pub fn payload<B: ipc::BinaryConvertable>() -> Result<B, BootError> {
 }
 
 pub fn register(hv_url: &str, control_url: &str, module_id: IpcModuleId) -> GuardedSocket<HypervisorServiceClient<NanoSocket>>{
-	let hypervisor_client = nanoipc::init_client::<HypervisorServiceClient<_>>(hv_url).unwrap();
+	let hypervisor_client = nanoipc::fast_client::<HypervisorServiceClient<_>>(hv_url).unwrap();
 	hypervisor_client.handshake().unwrap();
 	hypervisor_client.module_ready(module_id, control_url.to_owned());
 
@@ -73,7 +73,7 @@ pub fn register(hv_url: &str, control_url: &str, module_id: IpcModuleId) -> Guar
 pub fn dependency<C: WithSocket<NanoSocket>>(url: &str)
 	-> Result<GuardedSocket<C>, BootError>
 {
-	nanoipc::init_client::<C>(url).map_err(|socket_err| BootError::DependencyConnect(socket_err))
+	nanoipc::generic_client::<C>(url).map_err(|socket_err| BootError::DependencyConnect(socket_err))
 }
 
 pub fn main_thread() -> Arc<AtomicBool> {

parity/cli.rs

@@ -243,6 +243,8 @@ Snapshot Options:
                            index, hash, or 'latest'. Note that taking snapshots at
                            non-recent blocks will only work with --pruning archive
                            [default: latest]
+  --no-periodic-snapshot   Disable automated snapshots which usually occur once
+                           every 10000 blocks.
 
 Virtual Machine Options:
   --jitvm                  Enable the JIT VM.
@@ -382,6 +384,7 @@ pub struct Args {
 	pub flag_from: String,
 	pub flag_to: String,
 	pub flag_at: String,
+	pub flag_no_periodic_snapshot: bool,
 	pub flag_format: Option<String>,
 	pub flag_jitvm: bool,
 	pub flag_log_file: Option<String>,

parity/configuration.rs

@@ -226,6 +226,7 @@ impl Configuration {
 				ui: self.args.cmd_ui,
 				name: self.args.flag_identity,
 				custom_bootnodes: self.args.flag_bootnodes.is_some(),
+				no_periodic_snapshot: self.args.flag_no_periodic_snapshot,
 			};
 			Cmd::Run(run_cmd)
 		};
@@ -802,6 +803,7 @@ mod tests {
 			ui: false,
 			name: "".into(),
 			custom_bootnodes: false,
+			no_periodic_snapshot: false,
 		}));
 	}
 

parity/dir.rs

@@ -52,10 +52,16 @@ impl Directories {
 		Ok(())
 	}
 
-	/// Get the root path for database
+	/// Get the chain's root path.
-	pub fn db_version_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf {
+	pub fn chain_path(&self, genesis_hash: H256, fork_name: Option<&String>) -> PathBuf {
 		let mut dir = Path::new(&self.db).to_path_buf();
 		dir.push(format!("{:?}{}", H64::from(genesis_hash), fork_name.map(|f| format!("-{}", f)).unwrap_or_default()));
+		dir
+	}
+
+	/// Get the root path for database
+	pub fn db_version_path(&self, genesis_hash: H256, fork_name: Option<&String>, pruning: Algorithm) -> PathBuf {
+		let mut dir = self.chain_path(genesis_hash, fork_name);
 		dir.push(format!("v{}-sec-{}", LEGACY_CLIENT_DB_VER_STR, pruning.as_internal_name_str()));
 		dir
 	}
@@ -67,6 +73,13 @@ impl Directories {
 		dir
 	}
 
+	/// Get the path for the snapshot directory given the genesis hash and fork name.
+	pub fn snapshot_path(&self, genesis_hash: H256, fork_name: Option<&String>) -> PathBuf {
+		let mut dir = self.chain_path(genesis_hash, fork_name);
+		dir.push("snapshot");
+		dir
+	}
+
 	/// Get the ipc sockets path
 	pub fn ipc_path(&self) -> PathBuf {
 		let mut dir = Path::new(&self.db).to_path_buf();

parity/io_handler.rs

@@ -15,6 +15,7 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
 use ethcore::client::Client;
 use ethcore::service::ClientIoMessage;
 use ethsync::{SyncProvider, ManageNetwork};
@@ -31,6 +32,7 @@ pub struct ClientIoHandler {
 	pub net: Arc<ManageNetwork>,
 	pub accounts: Arc<AccountProvider>,
 	pub info: Arc<Informant>,
+	pub shutdown: Arc<AtomicBool>
 }
 
 impl IoHandler<ClientIoMessage> for ClientIoHandler {
@@ -39,7 +41,7 @@ impl IoHandler<ClientIoMessage> for ClientIoHandler {
 	}
 
 	fn timeout(&self, _io: &IoContext<ClientIoMessage>, timer: TimerToken) {
-		if let INFO_TIMER = timer {
+		if timer == INFO_TIMER && !self.shutdown.load(Ordering::SeqCst) {
 			self.info.tick();
 		}
 	}

parity/main.rs

@@ -99,9 +99,11 @@ mod modules;
 mod account;
 mod blockchain;
 mod presale;
-mod run;
-mod sync;
 mod snapshot;
+mod run;
+#[cfg(feature="ipc")]
+mod sync;
+#[cfg(feature="ipc")]
 mod boot;
 
 #[cfg(feature="stratum")]
@@ -158,10 +160,24 @@ mod stratum_optional {
 	}
 }
 
-fn main() {
+#[cfg(not(feature="ipc"))]
+fn sync_main() -> bool {
+	false
+}
+
+#[cfg(feature="ipc")]
+fn sync_main() -> bool {
 	// just redirect to the sync::main()
 	if std::env::args().nth(1).map_or(false, |arg| arg == "sync") {
 		sync::main();
+		true
+	} else {
+		false
+	}
+}
+
+fn main() {
+	if sync_main() {
 		return;
 	}
 

parity/modules.rs

@@ -18,6 +18,7 @@ use std::sync::Arc;
 use ethcore::client::BlockChainClient;
 use hypervisor::Hypervisor;
 use ethsync::{SyncConfig, NetworkConfiguration, NetworkError};
+use ethcore::snapshot::SnapshotService;
 #[cfg(not(feature="ipc"))]
 use self::no_ipc_deps::*;
 #[cfg(feature="ipc")]
@@ -25,10 +26,12 @@ use self::ipc_deps::*;
 use ethcore_logger::Config as LogConfig;
 use std::path::Path;
 
+#[cfg(feature="ipc")]
 pub mod service_urls {
 	use std::path::PathBuf;
 
 	pub const CLIENT: &'static str = "parity-chain.ipc";
+	pub const SNAPSHOT: &'static str = "parity-snapshot.ipc";
 	pub const SYNC: &'static str = "parity-sync.ipc";
 	pub const SYNC_NOTIFY: &'static str = "parity-sync-notify.ipc";
 	pub const NETWORK_MANAGER: &'static str = "parity-manage-net.ipc";
@@ -68,7 +71,7 @@ mod ipc_deps {
 	pub use ethsync::{SyncClient, NetworkManagerClient, ServiceConfiguration};
 	pub use ethcore::client::ChainNotifyClient;
 	pub use hypervisor::{SYNC_MODULE_ID, BootArgs, HYPERVISOR_IPC_URL};
-	pub use nanoipc::{GuardedSocket, NanoSocket, init_client};
+	pub use nanoipc::{GuardedSocket, NanoSocket, generic_client, fast_client};
 	pub use ipc::IpcSocket;
 	pub use ipc::binary::serialize;
 }
@@ -119,6 +122,7 @@ pub fn sync
 		sync_cfg: SyncConfig,
 		net_cfg: NetworkConfiguration,
 		_client: Arc<BlockChainClient>,
+		_snapshot_service: Arc<SnapshotService>,
 		log_settings: &LogConfig,
 	)
 	-> Result<SyncModules, NetworkError>
@@ -130,11 +134,11 @@ pub fn sync
 	hypervisor.start();
 	hypervisor.wait_for_startup();
 
-	let sync_client = init_client::<SyncClient<_>>(
+	let sync_client = generic_client::<SyncClient<_>>(
 		&service_urls::with_base(&hypervisor.io_path, service_urls::SYNC)).unwrap();
-	let notify_client = init_client::<ChainNotifyClient<_>>(
+	let notify_client = generic_client::<ChainNotifyClient<_>>(
 		&service_urls::with_base(&hypervisor.io_path, service_urls::SYNC_NOTIFY)).unwrap();
-	let manage_client = init_client::<NetworkManagerClient<_>>(
+	let manage_client = generic_client::<NetworkManagerClient<_>>(
 		&service_urls::with_base(&hypervisor.io_path, service_urls::NETWORK_MANAGER)).unwrap();
 
 	*hypervisor_ref = Some(hypervisor);
@@ -148,10 +152,11 @@ pub fn sync
 		sync_cfg: SyncConfig,
 		net_cfg: NetworkConfiguration,
 		client: Arc<BlockChainClient>,
+		snapshot_service: Arc<SnapshotService>,
 		_log_settings: &LogConfig,
 	)
 	-> Result<SyncModules, NetworkError>
 {
-	let eth_sync = try!(EthSync::new(sync_cfg, client, net_cfg));
+	let eth_sync = try!(EthSync::new(sync_cfg, client, snapshot_service, net_cfg));
 	Ok((eth_sync.clone() as Arc<SyncProvider>, eth_sync.clone() as Arc<ManageNetwork>, eth_sync.clone() as Arc<ChainNotify>))
 }

parity/params.rs

@@ -17,7 +17,7 @@
 use std::str::FromStr;
 use std::fs;
 use std::time::Duration;
-use util::{contents, H256, Address, U256, version_data};
+use util::{H256, Address, U256, version_data};
 use util::journaldb::Algorithm;
 use ethcore::spec::Spec;
 use ethcore::ethereum;
@@ -61,7 +61,10 @@ impl SpecType {
 			SpecType::Testnet => Ok(ethereum::new_morden()),
 			SpecType::Olympic => Ok(ethereum::new_olympic()),
 			SpecType::Classic => Ok(ethereum::new_classic()),
-			SpecType::Custom(ref file) => Ok(Spec::load(&try!(contents(file).map_err(|_| "Could not load specification file."))))
+			SpecType::Custom(ref filename) => {
+				let file = try!(fs::File::open(filename).map_err(|_| "Could not load specification file."));
+				Spec::load(file)
+			}
 		}
 	}
 }

parity/run.rs

@@ -15,7 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::sync::{Arc, Mutex, Condvar};
-use std::path::Path;
 use std::io::ErrorKind;
 use ctrlc::CtrlC;
 use fdlimit::raise_fd_limit;
@@ -28,7 +27,8 @@ use ethcore::client::{Mode, Switch, DatabaseCompactionProfile, VMType, ChainNoti
 use ethcore::service::ClientService;
 use ethcore::account_provider::AccountProvider;
 use ethcore::miner::{Miner, MinerService, ExternalMiner, MinerOptions};
-use ethsync::SyncConfig;
+use ethcore::snapshot;
+use ethsync::{SyncConfig, SyncProvider};
 use informant::Informant;
 
 use rpc::{HttpServer, IpcServer, HttpConfiguration, IpcConfiguration};
@@ -46,6 +46,12 @@ use rpc_apis;
 use rpc;
 use url;
 
+// how often to take periodic snapshots.
+const SNAPSHOT_PERIOD: u64 = 10000;
+
+// how many blocks to wait before starting a periodic snapshot.
+const SNAPSHOT_HISTORY: u64 = 500;
+
 #[derive(Debug, PartialEq)]
 pub struct RunCmd {
 	pub cache_config: CacheConfig,
@@ -77,6 +83,7 @@ pub struct RunCmd {
 	pub ui: bool,
 	pub name: String,
 	pub custom_bootnodes: bool,
+	pub no_periodic_snapshot: bool,
 }
 
 pub fn execute(cmd: RunCmd) -> Result<(), String> {
@@ -102,8 +109,9 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
 	// select pruning algorithm
 	let algorithm = cmd.pruning.to_algorithm(&cmd.dirs, genesis_hash, fork_name.as_ref());
 
-	// prepare client_path
+	// prepare client and snapshot paths.
 	let client_path = cmd.dirs.client_path(genesis_hash, fork_name.as_ref(), algorithm);
+	let snapshot_path = cmd.dirs.snapshot_path(genesis_hash, fork_name.as_ref());
 
 	// execute upgrades
 	try!(execute_upgrades(&cmd.dirs, genesis_hash, fork_name.as_ref(), algorithm, cmd.compaction.compaction_profile()));
@@ -163,14 +171,15 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
 	}
 
 	// create supervisor
-	let mut hypervisor = modules::hypervisor(Path::new(&cmd.dirs.ipc_path()));
+	let mut hypervisor = modules::hypervisor(&cmd.dirs.ipc_path());
 
 	// create client service.
 	let service = try!(ClientService::start(
 		client_config,
 		&spec,
-		Path::new(&client_path),
+		&client_path,
-		Path::new(&cmd.dirs.ipc_path()),
+		&snapshot_path,
+		&cmd.dirs.ipc_path(),
 		miner.clone(),
 	).map_err(|e| format!("Client service error: {:?}", e)));
 
@@ -179,13 +188,14 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
 
 	// take handle to client
 	let client = service.client();
+	let snapshot_service = service.snapshot_service();
 
 	// create external miner
 	let external_miner = Arc::new(ExternalMiner::default());
 
 	// create sync object
 	let (sync_provider, manage_network, chain_notify) = try!(modules::sync(
-		&mut hypervisor, sync_config, net_conf.into(), client.clone(), &cmd.logger_config,
+		&mut hypervisor, sync_config, net_conf.into(), client.clone(), snapshot_service, &cmd.logger_config,
 	).map_err(|e| format!("Sync error: {}", e)));
 
 	service.add_notify(chain_notify.clone());
@@ -247,8 +257,27 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
 		sync: sync_provider.clone(),
 		net: manage_network.clone(),
 		accounts: account_provider.clone(),
+		shutdown: Default::default(),
 	});
-	service.register_io_handler(io_handler).expect("Error registering IO handler");
+	service.register_io_handler(io_handler.clone()).expect("Error registering IO handler");
+
+	// the watcher must be kept alive.
+	let _watcher = match cmd.no_periodic_snapshot {
+		true => None,
+		false => {
+			let sync = sync_provider.clone();
+			let watcher = Arc::new(snapshot::Watcher::new(
+				service.client(),
+				move || sync.status().is_major_syncing(),
+				service.io().channel(),
+				SNAPSHOT_PERIOD,
+				SNAPSHOT_HISTORY,
+			));
+
+			service.add_notify(watcher.clone());
+			Some(watcher)
+		},
+	};
 
 	// start ui
 	if cmd.ui {
@@ -261,6 +290,11 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> {
 	// Handle exit
 	wait_for_exit(panic_handler, http_server, ipc_server, dapps_server, signer_server);
 
+	// to make sure timer does not spawn requests while shutdown is in progress
+	io_handler.shutdown.store(true, ::std::sync::atomic::Ordering::SeqCst);
+	// just Arc is dropping here, to allow other reference release in its default time
+	drop(io_handler);
+
 	// hypervisor should be shutdown first while everything still works and can be
 	// terminated gracefully
 	drop(hypervisor);

parity/snapshot.rs

@@ -82,8 +82,9 @@ impl SnapshotCommand {
 		// select pruning algorithm
 		let algorithm = self.pruning.to_algorithm(&self.dirs, genesis_hash, spec.fork_name.as_ref());
 
-		// prepare client_path
+		// prepare client and snapshot paths.
 		let client_path = self.dirs.client_path(genesis_hash, spec.fork_name.as_ref(), algorithm);
+		let snapshot_path = self.dirs.snapshot_path(genesis_hash, spec.fork_name.as_ref());
 
 		// execute upgrades
 		try!(execute_upgrades(&self.dirs, genesis_hash, spec.fork_name.as_ref(), algorithm, self.compaction.compaction_profile()));
@@ -94,8 +95,9 @@ impl SnapshotCommand {
 		let service = try!(ClientService::start(
 			client_config,
 			&spec,
-			Path::new(&client_path),
+			&client_path,
-			Path::new(&self.dirs.ipc_path()),
+			&snapshot_path,
+			&self.dirs.ipc_path(),
 			Arc::new(Miner::with_spec(&spec))
 		).map_err(|e| format!("Client service error: {:?}", e)));
 
@@ -129,10 +131,9 @@ impl SnapshotCommand {
 
 		let informant_handle = snapshot.clone();
 		::std::thread::spawn(move || {
- 			while let RestorationStatus::Ongoing = informant_handle.status() {
+ 			while let RestorationStatus::Ongoing { state_chunks_done, block_chunks_done } = informant_handle.status() {
- 				let (state_chunks, block_chunks) = informant_handle.chunks_done();
  				info!("Processed {}/{} state chunks and {}/{} block chunks.",
- 					state_chunks, num_state, block_chunks, num_blocks);
+ 					state_chunks_done, num_state, block_chunks_done, num_blocks);
 
  				::std::thread::sleep(Duration::from_secs(5));
  			}
@@ -161,7 +162,7 @@ impl SnapshotCommand {
 		}
 
 		match snapshot.status() {
-			RestorationStatus::Ongoing => Err("Snapshot file is incomplete and missing chunks.".into()),
+			RestorationStatus::Ongoing { .. } => Err("Snapshot file is incomplete and missing chunks.".into()),
 			RestorationStatus::Failed => Err("Snapshot restoration failed.".into()),
 			RestorationStatus::Inactive => {
 				info!("Restoration complete.");

parity/sync.rs

@@ -20,6 +20,7 @@ use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use hypervisor::{SYNC_MODULE_ID, HYPERVISOR_IPC_URL, ControlService};
 use ethcore::client::{RemoteClient, ChainNotify};
+use ethcore::snapshot::{RemoteSnapshotService};
 use ethsync::{SyncProvider, EthSync, ManageNetwork, ServiceConfiguration};
 use modules::service_urls;
 use boot;
@@ -45,8 +46,9 @@ pub fn main() {
 		.unwrap_or_else(|e| panic!("Fatal: error reading boot arguments ({:?})", e));
 
 	let remote_client = dependency!(RemoteClient, &service_urls::with_base(&service_config.io_path, service_urls::CLIENT));
+	let remote_snapshot = dependency!(RemoteSnapshotService, &service_urls::with_base(&service_config.io_path, service_urls::SNAPSHOT));
 
-	let sync = EthSync::new(service_config.sync, remote_client.service().clone(), service_config.net).unwrap();
+	let sync = EthSync::new(service_config.sync, remote_client.service().clone(), remote_snapshot.service().clone(), service_config.net).unwrap();
 
 	let _ = boot::main_thread();
 	let service_stop = Arc::new(AtomicBool::new(false));

rpc/src/v1/impls/eth.rs

@@ -254,7 +254,8 @@ impl<C, S: ?Sized, M, EM> Eth for EthClient<C, S, M, EM> where
 		let status = take_weak!(self.sync).status();
 		let res = match status.state {
 			SyncState::Idle => SyncStatus::None,
-			SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead => {
+			SyncState::Waiting | SyncState::Blocks | SyncState::NewBlocks | SyncState::ChainHead
+				| SyncState::SnapshotManifest | SyncState::SnapshotData | SyncState::SnapshotWaiting => {
 				let current_block = U256::from(take_weak!(self.client).chain_info().best_block_number);
 				let highest_block = U256::from(status.highest_block_number.unwrap_or(status.start_block_number));
 

rpc/src/v1/tests/eth.rs

@@ -108,7 +108,16 @@ impl EthTester {
 		let dir = RandomTempPath::new();
 		let account_provider = account_provider();
 		let miner_service = miner_service(&spec, account_provider.clone());
-		let client = Client::new(ClientConfig::default(), &spec, dir.as_path(), miner_service.clone(), IoChannel::disconnected()).unwrap();
+
+		let db_config = ::util::kvdb::DatabaseConfig::with_columns(::ethcore::db::NUM_COLUMNS);
+		let client = Client::new(
+			ClientConfig::default(),
+			&spec,
+			dir.as_path(),
+			miner_service.clone(),
+			IoChannel::disconnected(),
+			&db_config
+		).unwrap();
 		let sync_provider = sync_provider();
 		let external_miner = Arc::new(ExternalMiner::default());
 
@@ -286,7 +295,7 @@ const POSITIVE_NONCE_SPEC: &'static [u8] = br#"{
 #[test]
 fn eth_transaction_count() {
 	let secret = "8a283037bb19c4fed7b1c569e40c7dcff366165eb869110a1b11532963eb9cb2".into();
-	let tester = EthTester::from_spec(Spec::load(TRANSACTION_COUNT_SPEC));
+	let tester = EthTester::from_spec(Spec::load(TRANSACTION_COUNT_SPEC).expect("invalid chain spec"));
 	let address = tester.accounts.insert_account(secret, "").unwrap();
 	tester.accounts.unlock_account_permanently(address, "".into()).unwrap();
 
@@ -412,7 +421,7 @@ fn verify_transaction_counts(name: String, chain: BlockChain) {
 
 #[test]
 fn starting_nonce_test() {
-	let tester = EthTester::from_spec(Spec::load(POSITIVE_NONCE_SPEC));
+	let tester = EthTester::from_spec(Spec::load(POSITIVE_NONCE_SPEC).expect("invalid chain spec"));
 	let address = Address::from(10);
 
 	let sample = tester.handler.handle_request_sync(&(r#"

rpc/src/v1/tests/helpers/sync_provider.rs

@@ -49,6 +49,8 @@ impl TestSyncProvider {
 				num_peers: config.num_peers,
 				num_active_peers: 0,
 				mem_used: 0,
+				num_snapshot_chunks: 0,
+				snapshot_chunks_done: 0,
 			}),
 		}
 	}

sync/src/api.rs

@@ -20,6 +20,7 @@ use network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId,
 use util::{U256, H256};
 use io::{TimerToken};
 use ethcore::client::{BlockChainClient, ChainNotify};
+use ethcore::snapshot::SnapshotService;
 use ethcore::header::BlockNumber;
 use sync_io::NetSyncIo;
 use chain::{ChainSync, SyncStatus};
@@ -71,12 +72,12 @@ pub struct EthSync {
 
 impl EthSync {
 	/// Creates and register protocol with the network service
-	pub fn new(config: SyncConfig, chain: Arc<BlockChainClient>, network_config: NetworkConfiguration) -> Result<Arc<EthSync>, NetworkError> {
+	pub fn new(config: SyncConfig, chain: Arc<BlockChainClient>, snapshot_service: Arc<SnapshotService>, network_config: NetworkConfiguration) -> Result<Arc<EthSync>, NetworkError> {
 		let chain_sync = ChainSync::new(config, &*chain);
 		let service = try!(NetworkService::new(try!(network_config.into_basic())));
 		let sync = Arc::new(EthSync{
 			network: service,
-			handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain }),
+			handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain, snapshot_service: snapshot_service }),
 		});
 
 		Ok(sync)
@@ -93,8 +94,10 @@ impl SyncProvider for EthSync {
 }
 
 struct SyncProtocolHandler {
-	/// Shared blockchain client. TODO: this should evetually become an IPC endpoint
+	/// Shared blockchain client.
 	chain: Arc<BlockChainClient>,
+	/// Shared snapshot service.
+	snapshot_service: Arc<SnapshotService>,
 	/// Sync strategy
 	sync: RwLock<ChainSync>,
 }
@@ -105,21 +108,21 @@ impl NetworkProtocolHandler for SyncProtocolHandler {
 	}
 
 	fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) {
-		ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain), *peer, packet_id, data);
+		ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer, packet_id, data);
 	}
 
 	fn connected(&self, io: &NetworkContext, peer: &PeerId) {
-		self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain), *peer);
+		self.sync.write().on_peer_connected(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer);
 	}
 
 	fn disconnected(&self, io: &NetworkContext, peer: &PeerId) {
-		self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain), *peer);
+		self.sync.write().on_peer_aborting(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service), *peer);
 	}
 
 	fn timeout(&self, io: &NetworkContext, _timer: TimerToken) {
-		self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain));
+		self.sync.write().maintain_peers(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
-		self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain));
+		self.sync.write().maintain_sync(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
-		self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain));
+		self.sync.write().propagate_new_transactions(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service));
 	}
 }
 
@@ -133,7 +136,7 @@ impl ChainNotify for EthSync {
 		_duration: u64)
 	{
 		self.network.with_context(ETH_PROTOCOL, |context| {
-			let mut sync_io = NetSyncIo::new(context, &*self.handler.chain);
+			let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service);
 			self.handler.sync.write().chain_new_blocks(
 				&mut sync_io,
 				&imported,
@@ -146,7 +149,7 @@ impl ChainNotify for EthSync {
 
 	fn start(&self) {
 		self.network.start().unwrap_or_else(|e| warn!("Error starting network: {:?}", e));
-		self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8])
+		self.network.register_protocol(self.handler.clone(), ETH_PROTOCOL, &[62u8, 63u8, 64u8])
 			.unwrap_or_else(|e| warn!("Error registering ethereum protocol: {:?}", e));
 	}
 
@@ -202,7 +205,7 @@ impl ManageNetwork for EthSync {
 
 	fn stop_network(&self) {
 		self.network.with_context(ETH_PROTOCOL, |context| {
-			let mut sync_io = NetSyncIo::new(context, &*self.handler.chain);
+			let mut sync_io = NetSyncIo::new(context, &*self.handler.chain, &*self.handler.snapshot_service);
 			self.handler.sync.write().abort(&mut sync_io);
 		});
 		self.stop();

sync/src/chain.rs

@@ -96,17 +96,19 @@ use ethcore::header::{BlockNumber, Header as BlockHeader};
 use ethcore::client::{BlockChainClient, BlockStatus, BlockID, BlockChainInfo, BlockImportError};
 use ethcore::error::*;
 use ethcore::block::Block;
+use ethcore::snapshot::{ManifestData, RestorationStatus};
 use sync_io::SyncIo;
 use time;
 use super::SyncConfig;
 use blocks::BlockCollection;
+use snapshot::{Snapshot, ChunkType};
 use rand::{thread_rng, Rng};
 
 known_heap_size!(0, PeerInfo);
 
 type PacketDecodeError = DecoderError;
 
-const PROTOCOL_VERSION: u8 = 63u8;
+const PROTOCOL_VERSION: u8 = 64u8;
 const MAX_BODIES_TO_SEND: usize = 256;
 const MAX_HEADERS_TO_SEND: usize = 512;
 const MAX_NODE_DATA_TO_SEND: usize = 1024;
@@ -136,14 +138,26 @@ const GET_NODE_DATA_PACKET: u8 = 0x0d;
 const NODE_DATA_PACKET: u8 = 0x0e;
 const GET_RECEIPTS_PACKET: u8 = 0x0f;
 const RECEIPTS_PACKET: u8 = 0x10;
+const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11;
+const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12;
+const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13;
+const SNAPSHOT_DATA_PACKET: u8 = 0x14;
 
 const HEADERS_TIMEOUT_SEC: f64 = 15f64;
 const BODIES_TIMEOUT_SEC: f64 = 5f64;
 const FORK_HEADER_TIMEOUT_SEC: f64 = 3f64;
+const SNAPSHOT_MANIFEST_TIMEOUT_SEC: f64 = 3f64;
+const SNAPSHOT_DATA_TIMEOUT_SEC: f64 = 10f64;
 
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
 /// Sync state
 pub enum SyncState {
+	/// Waiting for pv64 peers to start snapshot syncing
+	SnapshotManifest,
+	/// Downloading snapshot data
+	SnapshotData,
+	/// Waiting for snapshot restoration to complete
+	SnapshotWaiting,
 	/// Downloading subchain heads
 	ChainHead,
 	/// Initial chain sync complete. Waiting for new packets
@@ -177,10 +191,14 @@ pub struct SyncStatus {
 	pub blocks_received: BlockNumber,
 	/// Total number of connected peers
 	pub num_peers: usize,
-	/// Total number of active peers
+	/// Total number of active peers.
 	pub num_active_peers: usize,
-	/// Heap memory used in bytes
+	/// Heap memory used in bytes.
 	pub mem_used: usize,
+	/// Snapshot chunks
+	pub num_snapshot_chunks: usize,
+	/// Snapshot chunks downloaded
+	pub snapshot_chunks_done: usize,
 }
 
 impl SyncStatus {
@@ -207,6 +225,8 @@ enum PeerAsking {
 	BlockHeaders,
 	BlockBodies,
 	Heads,
+	SnapshotManifest,
+	SnapshotData,
 }
 
 #[derive(Clone, Eq, PartialEq)]
@@ -240,6 +260,8 @@ struct PeerInfo {
 	asking_blocks: Vec<H256>,
 	/// Holds requested header hash if currently requesting block header by hash
 	asking_hash: Option<H256>,
+	/// Holds requested snapshot chunk hash if any.
+	asking_snapshot_data: Option<H256>,
 	/// Request timestamp
 	ask_time: f64,
 	/// Holds a set of transactions recently sent to this peer to avoid spamming.
@@ -248,6 +270,10 @@ struct PeerInfo {
 	expired: bool,
 	/// Peer fork confirmation status
 	confirmation: ForkConfirmation,
+	/// Best snapshot hash
+	snapshot_hash: Option<H256>,
+	/// Best snapshot block number
+	snapshot_number: Option<BlockNumber>,
 }
 
 impl PeerInfo {
@@ -293,6 +319,8 @@ pub struct ChainSync {
 	network_id: U256,
 	/// Optional fork block to check
 	fork_block: Option<(BlockNumber, H256)>,
+	/// Snapshot downloader.
+	snapshot: Snapshot,
 }
 
 type RlpResponseResult = Result<Option<(PacketId, RlpStream)>, PacketDecodeError>;
@@ -301,8 +329,8 @@ impl ChainSync {
 	/// Create a new instance of syncing strategy.
 	pub fn new(config: SyncConfig, chain: &BlockChainClient) -> ChainSync {
 		let chain = chain.chain_info();
-		let mut sync = ChainSync {
+		ChainSync {
-			state: SyncState::ChainHead,
+			state: SyncState::Idle,
 			starting_block: chain.best_block_number,
 			highest_block: None,
 			last_imported_block: chain.best_block_number,
@@ -317,16 +345,15 @@ impl ChainSync {
 			_max_download_ahead_blocks: max(MAX_HEADERS_TO_REQUEST, config.max_download_ahead_blocks),
 			network_id: config.network_id,
 			fork_block: config.fork_block,
-		};
+			snapshot: Snapshot::new(),
-		sync.reset();
+		}
-		sync
 	}
 
 	/// @returns Synchonization status
 	pub fn status(&self) -> SyncStatus {
 		SyncStatus {
 			state: self.state.clone(),
-			protocol_version: 63,
+			protocol_version: if self.state == SyncState::SnapshotData { 64 } else { 63 },
 			network_id: self.network_id,
 			start_block_number: self.starting_block,
 			last_imported_block_number: Some(self.last_imported_block),
@@ -335,6 +362,8 @@ impl ChainSync {
 			blocks_total: match self.highest_block { Some(x) if x > self.starting_block => x - self.starting_block, _ => 0 },
 			num_peers: self.peers.values().filter(|p| p.is_allowed()).count(),
 			num_active_peers: self.peers.values().filter(|p| p.is_allowed() && p.asking != PeerAsking::Nothing).count(),
+			num_snapshot_chunks: self.snapshot.total_chunks(),
+			snapshot_chunks_done: self.snapshot.done_chunks(),
 			mem_used:
 				self.blocks.heap_size()
 				+ self.peers.heap_size_of_children()
@@ -350,8 +379,13 @@ impl ChainSync {
 
 	#[cfg_attr(feature="dev", allow(for_kv_map))] // Because it's not possible to get `values_mut()`
 	/// Reset sync. Clear all downloaded data but keep the queue
-	fn reset(&mut self) {
+	fn reset(&mut self, io: &mut SyncIo) {
 		self.blocks.clear();
+		self.snapshot.clear();
+		if self.state == SyncState::SnapshotData {
+			debug!(target:"sync", "Aborting snapshot restore");
+			io.snapshot_service().abort_restore();
+		}
 		for (_, ref mut p) in &mut self.peers {
 			p.asking_blocks.clear();
 			p.asking_hash = None;
@@ -368,7 +402,7 @@ impl ChainSync {
 	/// Restart sync
 	pub fn restart(&mut self, io: &mut SyncIo) {
 		trace!(target: "sync", "Restarting");
-		self.reset();
+		self.reset(io);
 		self.start_sync_round(io);
 		self.continue_sync(io);
 	}
@@ -380,13 +414,19 @@ impl ChainSync {
 		if self.active_peers.is_empty() {
 			trace!(target: "sync", "No more active peers");
 			if self.state == SyncState::ChainHead {
-				self.complete_sync();
+				self.complete_sync(io);
 			} else {
 				self.restart(io);
 			}
 		}
 	}
 
+	fn start_snapshot_sync(&mut self, io: &mut SyncIo, peer_id: PeerId) {
+		self.snapshot.clear();
+		self.request_snapshot_manifest(io, peer_id);
+		self.state = SyncState::SnapshotManifest;
+	}
+  
 	/// Restart sync after bad block has been detected. May end up re-downloading up to QUEUE_SIZE blocks
 	fn restart_on_bad_block(&mut self, io: &mut SyncIo) {
 		// Do not assume that the block queue/chain still has our last_imported_block
@@ -398,8 +438,9 @@ impl ChainSync {
 
 	/// Called by peer to report status
 	fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
+		let protocol_version: u32 = try!(r.val_at(0));
 		let peer = PeerInfo {
-			protocol_version: try!(r.val_at(0)),
+			protocol_version: protocol_version,
 			network_id: try!(r.val_at(1)),
 			difficulty: Some(try!(r.val_at(2))),
 			latest_hash: try!(r.val_at(3)),
@@ -412,6 +453,9 @@ impl ChainSync {
 			last_sent_transactions: HashSet::new(),
 			expired: false,
 			confirmation: if self.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed },
+			asking_snapshot_data: None,
+			snapshot_hash: if protocol_version == 64 { Some(try!(r.val_at(5))) } else { None },
+			snapshot_number: if protocol_version == 64 { Some(try!(r.val_at(6))) } else { None },
 		};
 
 		trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{})", peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis);
@@ -749,6 +793,96 @@ impl ChainSync {
 		Ok(())
 	}
 
+	/// Called when snapshot manifest is downloaded from a peer.
+	fn on_snapshot_manifest(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
+		if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		self.clear_peer_download(peer_id);
+		if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || self.state != SyncState::SnapshotManifest {
+			trace!(target: "sync", "{}: Ignored unexpected manifest", peer_id);
+			self.continue_sync(io);
+			return Ok(());
+		}
+
+		let manifest_rlp = try!(r.at(0));
+		let manifest = match ManifestData::from_rlp(&manifest_rlp.as_raw()) {
+			Err(e) => {
+				trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e);
+				io.disconnect_peer(peer_id);
+				self.continue_sync(io);
+				return Ok(());
+			}
+			Ok(manifest) => manifest,
+		};
+		self.snapshot.reset_to(&manifest, &manifest_rlp.as_raw().sha3());
+		io.snapshot_service().begin_restore(manifest);
+		self.state = SyncState::SnapshotData;
+
+		// give a task to the same peer first.
+		self.sync_peer(io, peer_id, false);
+		// give tasks to other peers
+		self.continue_sync(io);
+		Ok(())
+	}
+
+	/// Called when snapshot data is downloaded from a peer.
+	fn on_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> {
+		if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) {
+			trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id);
+			return Ok(());
+		}
+		self.clear_peer_download(peer_id);
+		if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || self.state != SyncState::SnapshotData {
+			trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id);
+			self.continue_sync(io);
+			return Ok(());
+		}
+
+		// check service status
+		match io.snapshot_service().status() {
+			RestorationStatus::Inactive | RestorationStatus::Failed => {
+				trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id);
+				self.state = SyncState::Idle;
+				self.snapshot.clear();
+				self.continue_sync(io);
+				return Ok(());
+			},
+			RestorationStatus::Ongoing { .. } => {
+				trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id);
+			},
+		}
+
+		let snapshot_data: Bytes = try!(r.val_at(0));
+		match self.snapshot.validate_chunk(&snapshot_data) {
+			Ok(ChunkType::Block(hash)) => {
+				trace!(target: "sync", "{}: Processing block chunk", peer_id);
+				io.snapshot_service().restore_block_chunk(hash, snapshot_data);
+			}
+			Ok(ChunkType::State(hash)) => {
+				trace!(target: "sync", "{}: Processing state chunk", peer_id);
+				io.snapshot_service().restore_state_chunk(hash, snapshot_data);
+			}
+			Err(()) => {
+				trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id);
+				io.disconnect_peer(peer_id);
+				self.continue_sync(io);
+				return Ok(());
+			}
+		}
+
+		if self.snapshot.is_complete() {
+			// wait for snapshot restoration process to complete
+			self.state = SyncState::SnapshotWaiting;
+		}
+		// give a task to the same peer first.
+		self.sync_peer(io, peer_id, false);
+		// give tasks to other peers
+		self.continue_sync(io);
+		Ok(())
+	}
+
 	/// Called by peer when it is disconnecting
 	pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) {
 		trace!(target: "sync", "== Disconnecting {}: {}", peer, io.peer_info(peer));
@@ -764,7 +898,7 @@ impl ChainSync {
 	/// Called when a new peer is connected
 	pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) {
 		trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer));
-		if let Err(e) = self.send_status(io) {
+		if let Err(e) = self.send_status(io, peer) {
 			debug!(target:"sync", "Error sending status request: {:?}", e);
 			io.disable_peer(peer);
 		}
@@ -772,24 +906,27 @@ impl ChainSync {
 
 	/// Resume downloading
 	fn continue_sync(&mut self, io: &mut SyncIo) {
-		let mut peers: Vec<(PeerId, U256)> = self.peers.iter().filter_map(|(k, p)|
+		let mut peers: Vec<(PeerId, U256, u32)> = self.peers.iter().filter_map(|(k, p)|
-			if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero))) } else { None }).collect();
+			if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero), p.protocol_version)) } else { None }).collect();
 		thread_rng().shuffle(&mut peers); //TODO: sort by rating
+		// prefer peers with higher protocol version
+		peers.sort_by(|&(_, _, ref v1), &(_, _, ref v2)| v1.cmp(v2));
 		trace!(target: "sync", "Syncing with {}/{} peers", self.active_peers.len(), peers.len());
-		for (p, _) in peers {
+		for (p, _, _) in peers {
 			if self.active_peers.contains(&p) {
 				self.sync_peer(io, p, false);
 			}
 		}
-		if self.state != SyncState::Waiting && !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) {
+		if self.state != SyncState::Waiting && self.state != SyncState::SnapshotWaiting
-			self.complete_sync();
+			&& !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.can_sync()) {
+			self.complete_sync(io);
 		}
 	}
 
 	/// Called after all blocks have been downloaded
-	fn complete_sync(&mut self) {
+	fn complete_sync(&mut self, io: &mut SyncIo) {
 		trace!(target: "sync", "Sync complete");
-		self.reset();
+		self.reset(io);
 		self.state = SyncState::Idle;
 	}
 
@@ -805,7 +942,7 @@ impl ChainSync {
 			trace!(target: "sync", "Skipping deactivated peer");
 			return;
 		}
-		let (peer_latest, peer_difficulty) = {
+		let (peer_latest, peer_difficulty, peer_snapshot_number, peer_snapshot_hash) = {
 			let peer = self.peers.get_mut(&peer_id).unwrap();
 			if peer.asking != PeerAsking::Nothing || !peer.can_sync() {
 				return;
@@ -814,7 +951,11 @@ impl ChainSync {
 				trace!(target: "sync", "Waiting for the block queue");
 				return;
 			}
-			(peer.latest_hash.clone(), peer.difficulty.clone())
+			if self.state == SyncState::SnapshotWaiting {
+				trace!(target: "sync", "Waiting for the snapshot restoration");
+				return;
+			}
+			(peer.latest_hash.clone(), peer.difficulty.clone(), peer.snapshot_number.as_ref().cloned(), peer.snapshot_hash.as_ref().cloned())
 		};
 		let chain_info = io.chain().chain_info();
 		let td = chain_info.pending_total_difficulty;
@@ -823,13 +964,18 @@ impl ChainSync {
 		if force || self.state == SyncState::NewBlocks || peer_difficulty.map_or(true, |pd| pd > syncing_difficulty) {
 			match self.state {
 				SyncState::Idle => {
-					if self.last_imported_block < chain_info.best_block_number {
+					// check if we can start snapshot sync with this peer
-						self.last_imported_block = chain_info.best_block_number;
+					if peer_snapshot_number.unwrap_or(0) > 0 && chain_info.best_block_number == 0 {
-						self.last_imported_hash = chain_info.best_block_hash;
+						self.start_snapshot_sync(io, peer_id);
+					} else {
+						if self.last_imported_block < chain_info.best_block_number {
+							self.last_imported_block = chain_info.best_block_number;
+							self.last_imported_hash = chain_info.best_block_hash;
+						}
+						trace!(target: "sync", "Starting sync with {}", peer_id);
+						self.start_sync_round(io);
+						self.sync_peer(io, peer_id, force);
 					}
-					trace!(target: "sync", "Starting sync with {}", peer_id);
-					self.start_sync_round(io);
-					self.sync_peer(io, peer_id, force);
 				},
 				SyncState::ChainHead => {
 					// Request subchain headers
@@ -843,8 +989,14 @@ impl ChainSync {
 					if io.chain().block_status(BlockID::Hash(peer_latest)) == BlockStatus::Unknown {
 						self.request_blocks(io, peer_id, false);
 					}
-				}
+				},
-				SyncState::Waiting => ()
+				SyncState::SnapshotData => {
+					if peer_snapshot_hash.is_some() && peer_snapshot_hash == self.snapshot.snapshot_hash() {
+						self.request_snapshot_data(io, peer_id);
+					}
+				},
+				SyncState::SnapshotManifest => (), //already downloading from other peer
+				SyncState::Waiting | SyncState::SnapshotWaiting => ()
 			}
 		}
 	}
@@ -903,6 +1055,16 @@ impl ChainSync {
 		}
 	}
 
+	/// Find some headers or blocks to download for a peer.
+	fn request_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId) {
+		self.clear_peer_download(peer_id);
+		// find chunk data to download
+		if let Some(hash) = self.snapshot.needed_chunk() {
+			self.peers.get_mut(&peer_id).unwrap().asking_snapshot_data = Some(hash.clone());
+			self.request_snapshot_chunk(io, peer_id, &hash);
+		}
+	}
+
 	/// Clear all blocks/headers marked as being downloaded by a peer.
 	fn clear_peer_download(&mut self, peer_id: PeerId) {
 		let peer = self.peers.get_mut(&peer_id).unwrap();
@@ -917,9 +1079,15 @@ impl ChainSync {
 					self.blocks.clear_body_download(b);
 				}
 			},
+			PeerAsking::SnapshotData => {
+				if let Some(hash) = peer.asking_snapshot_data {
+					self.snapshot.clear_chunk_download(&hash);
+				}
+			},
 			_ => (),
 		}
 		peer.asking_blocks.clear();
+		peer.asking_snapshot_data = None;
 	}
 
 	fn block_imported(&mut self, hash: &H256, number: BlockNumber, parent: &H256) {
@@ -1016,6 +1184,22 @@ impl ChainSync {
 		rlp.append(&if reverse {1u32} else {0u32});
 		self.send_request(sync, peer_id, asking, GET_BLOCK_HEADERS_PACKET, rlp.out());
 	}
+
+	/// Request snapshot manifest from a peer.
+	fn request_snapshot_manifest(&mut self, sync: &mut SyncIo, peer_id: PeerId) {
+		trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id);
+		let rlp = RlpStream::new_list(0);
+		self.send_request(sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out());
+	}
+
+	/// Request snapshot chunk from a peer.
+	fn request_snapshot_chunk(&mut self, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) {
+		trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk);
+		let mut rlp = RlpStream::new_list(1);
+		rlp.append(chunk);
+		self.send_request(sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out());
+	}
+
 	/// Request block bodies from a peer
 	fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec<H256>) {
 		let mut rlp = RlpStream::new_list(hashes.len());
@@ -1086,14 +1270,22 @@ impl ChainSync {
 	}
 
 	/// Send Status message
-	fn send_status(&mut self, io: &mut SyncIo) -> Result<(), NetworkError> {
+	fn send_status(&mut self, io: &mut SyncIo, peer: PeerId) -> Result<(), NetworkError> {
-		let mut packet = RlpStream::new_list(5);
+		let pv64 = io.eth_protocol_version(peer) >= 64;
+		let mut packet = RlpStream::new_list(if pv64 { 7 } else { 5 });
 		let chain = io.chain().chain_info();
 		packet.append(&(PROTOCOL_VERSION as u32));
 		packet.append(&self.network_id);
 		packet.append(&chain.total_difficulty);
 		packet.append(&chain.best_block_hash);
 		packet.append(&chain.genesis_hash);
+		if pv64 {
+			let manifest = io.snapshot_service().manifest();
+			let block_number = manifest.as_ref().map_or(0, |m| m.block_number);
+			let manifest_hash = manifest.map_or(H256::new(), |m| m.into_rlp().sha3());
+			packet.append(&manifest_hash);
+			packet.append(&block_number);
+		}
 		io.respond(STATUS_PACKET, packet.out())
 	}
 
@@ -1230,6 +1422,48 @@ impl ChainSync {
 		Ok(Some((RECEIPTS_PACKET, rlp_result)))
 	}
 
+	/// Respond to GetSnapshotManifest request
+	fn return_snapshot_manifest(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult {
+		let count = r.item_count();
+		trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id);
+		if count != 0 {
+			debug!(target: "sync", "Invalid GetSnapshotManifest request, ignoring.");
+			return Ok(None);
+		}
+		let rlp = match io.snapshot_service().manifest() {
+			Some(manifest) => {
+				trace!(target: "sync", "{} <- SnapshotManifest", peer_id);
+				let mut rlp = RlpStream::new_list(1);
+				rlp.append_raw(&manifest.into_rlp(), 1);
+				rlp
+			},
+			None => {
+				trace!(target: "sync", "{}: No manifest to return", peer_id);
+				let rlp = RlpStream::new_list(0);
+				rlp
+			}
+		};
+		Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp)))
+	}
+
+	/// Respond to GetSnapshotManifest request
+	fn return_snapshot_data(io: &SyncIo, r: &UntrustedRlp, peer_id: PeerId) -> RlpResponseResult {
+		let hash: H256 = try!(r.val_at(0));
+		trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash);
+		let rlp = match io.snapshot_service().chunk(hash) {
+			Some(data) => {
+				let mut rlp = RlpStream::new_list(1);
+				rlp.append(&data);
+				rlp
+			},
+			None => {
+				let rlp = RlpStream::new_list(0);
+				rlp
+			}
+		};
+		Ok(Some((SNAPSHOT_DATA_PACKET, rlp)))
+	}
+
 	fn return_rlp<FRlp, FError>(io: &mut SyncIo, rlp: &UntrustedRlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError>
 		where FRlp : Fn(&SyncIo, &UntrustedRlp, PeerId) -> RlpResponseResult,
 			FError : FnOnce(NetworkError) -> String
@@ -1266,6 +1500,14 @@ impl ChainSync {
 				ChainSync::return_node_data,
 				|e| format!("Error sending nodes: {:?}", e)),
 
+			GET_SNAPSHOT_MANIFEST_PACKET => ChainSync::return_rlp(io, &rlp, peer,
+				ChainSync::return_snapshot_manifest,
+				|e| format!("Error sending snapshot manifest: {:?}", e)),
+
+			GET_SNAPSHOT_DATA_PACKET => ChainSync::return_rlp(io, &rlp, peer,
+				ChainSync::return_snapshot_data,
+				|e| format!("Error sending snapshot data: {:?}", e)),
+
 			_ => {
 				sync.write().on_packet(io, peer, packet_id, data);
 				Ok(())
@@ -1289,6 +1531,8 @@ impl ChainSync {
 			BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp),
 			NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp),
 			NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp),
+			SNAPSHOT_MANIFEST_PACKET => self.on_snapshot_manifest(io, peer, &rlp),
+			SNAPSHOT_DATA_PACKET => self.on_snapshot_data(io, peer, &rlp),
 			_ => {
 				debug!(target: "sync", "Unknown packet {}", packet_id);
 				Ok(())
@@ -1308,6 +1552,8 @@ impl ChainSync {
 				PeerAsking::BlockBodies => (tick - peer.ask_time) > BODIES_TIMEOUT_SEC,
 				PeerAsking::Nothing => false,
 				PeerAsking::ForkHeader => (tick - peer.ask_time) > FORK_HEADER_TIMEOUT_SEC,
+				PeerAsking::SnapshotManifest => (tick - peer.ask_time) > SNAPSHOT_MANIFEST_TIMEOUT_SEC,
+				PeerAsking::SnapshotData => (tick - peer.ask_time) > SNAPSHOT_DATA_TIMEOUT_SEC,
 			};
 			if timeout {
 				trace!(target:"sync", "Timeout {}", peer_id);
@@ -1321,9 +1567,12 @@ impl ChainSync {
 	}
 
 	fn check_resume(&mut self, io: &mut SyncIo) {
-		if !io.chain().queue_info().is_full() && self.state == SyncState::Waiting {
+		if self.state == SyncState::Waiting && !io.chain().queue_info().is_full() && self.state == SyncState::Waiting {
 			self.state = SyncState::Blocks;
 			self.continue_sync(io);
+		} else if self.state == SyncState::SnapshotWaiting && io.snapshot_service().status() == RestorationStatus::Inactive {
+			self.state = SyncState::Idle;
+			self.continue_sync(io);
 		}
 	}
 
@@ -1559,6 +1808,7 @@ impl ChainSync {
 #[cfg(test)]
 mod tests {
 	use tests::helpers::*;
+	use tests::snapshot::TestSnapshotService;
 	use super::*;
 	use ::SyncConfig;
 	use util::*;
@@ -1612,7 +1862,8 @@ mod tests {
 	fn return_receipts_empty() {
 		let mut client = TestBlockChainClient::new();
 		let mut queue = VecDeque::new();
-		let io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let result = ChainSync::return_receipts(&io, &UntrustedRlp::new(&[0xc0]), 0);
 
@@ -1624,7 +1875,8 @@ mod tests {
 		let mut client = TestBlockChainClient::new();
 		let mut queue = VecDeque::new();
 		let sync = dummy_sync_with_peer(H256::new(), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let mut receipt_list = RlpStream::new_list(4);
 		receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
@@ -1679,7 +1931,8 @@ mod tests {
 		let hashes: Vec<_> = headers.iter().map(|h| HeaderView::new(h).sha3()).collect();
 
 		let mut queue = VecDeque::new();
-		let io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let unknown: H256 = H256::new();
 		let result = ChainSync::return_block_headers(&io, &UntrustedRlp::new(&make_hash_req(&unknown, 1, 0, false)), 0);
@@ -1717,7 +1970,8 @@ mod tests {
 		let mut client = TestBlockChainClient::new();
 		let mut queue = VecDeque::new();
 		let sync = dummy_sync_with_peer(H256::new(), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let mut node_list = RlpStream::new_list(3);
 		node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555"));
@@ -1758,6 +2012,9 @@ mod tests {
 				last_sent_transactions: HashSet::new(),
 				expired: false,
 				confirmation: super::ForkConfirmation::Confirmed,
+				snapshot_number: None,
+				snapshot_hash: None,
+				asking_snapshot_data: None,
 			});
 		sync
 	}
@@ -1769,7 +2026,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client);
 		let chain_info = client.chain_info();
-		let io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let lagging_peers = sync.get_lagging_peers(&chain_info, &io);
 
@@ -1800,7 +2058,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		let chain_info = client.chain_info();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let peers = sync.get_lagging_peers(&chain_info, &io);
 		let peer_count = sync.propagate_new_hashes(&chain_info, &mut io, &peers);
@@ -1820,7 +2079,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		let chain_info = client.chain_info();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 		let peers = sync.get_lagging_peers(&chain_info, &io);
 		let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[], &peers);
 
@@ -1840,7 +2100,8 @@ mod tests {
 		let hash = client.block_hash(BlockID::Number(99)).unwrap();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		let chain_info = client.chain_info();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 		let peers = sync.get_lagging_peers(&chain_info, &io);
 		let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[hash.clone()], &peers);
 
@@ -1859,7 +2120,8 @@ mod tests {
 		client.insert_transaction_to_queue();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
 		let mut queue = VecDeque::new();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 		let peer_count = sync.propagate_new_transactions(&mut io);
 		// Try to propagate same transactions for the second time
 		let peer_count2 = sync.propagate_new_transactions(&mut io);
@@ -1880,7 +2142,8 @@ mod tests {
 		client.insert_transaction_to_queue();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
 		let mut queue = VecDeque::new();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 		let peer_count = sync.propagate_new_transactions(&mut io);
 		sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[]);
 		// Try to propagate same transactions for the second time
@@ -1903,17 +2166,17 @@ mod tests {
 		client.insert_transaction_to_queue();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client);
 		let mut queue = VecDeque::new();
+		let mut ss = TestSnapshotService::new();
 		// should sent some
 		{
-
+			let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
-			let mut io = TestIo::new(&mut client, &mut queue, None);
 			let peer_count = sync.propagate_new_transactions(&mut io);
 			assert_eq!(1, io.queue.len());
 			assert_eq!(1, peer_count);
 		}
 		// Insert some more
 		client.insert_transaction_to_queue();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 		// Propagate new transactions
 		let peer_count2 = sync.propagate_new_transactions(&mut io);
 		// And now the peer should have all transactions
@@ -1939,7 +2202,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		//sync.have_common_block = true;
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let block = UntrustedRlp::new(&block_data);
 
@@ -1957,7 +2221,8 @@ mod tests {
 
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let block = UntrustedRlp::new(&block_data);
 
@@ -1972,7 +2237,8 @@ mod tests {
 		client.add_blocks(10, EachBlockWith::Uncle);
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let empty_data = vec![];
 		let block = UntrustedRlp::new(&empty_data);
@@ -1988,7 +2254,8 @@ mod tests {
 		client.add_blocks(10, EachBlockWith::Uncle);
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let hashes_data = get_dummy_hashes();
 		let hashes_rlp = UntrustedRlp::new(&hashes_data);
@@ -2004,7 +2271,8 @@ mod tests {
 		client.add_blocks(10, EachBlockWith::Uncle);
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let empty_hashes_data = vec![];
 		let hashes_rlp = UntrustedRlp::new(&empty_hashes_data);
@@ -2023,7 +2291,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		let chain_info = client.chain_info();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let peers = sync.get_lagging_peers(&chain_info, &io);
 		sync.propagate_new_hashes(&chain_info, &mut io, &peers);
@@ -2042,7 +2311,8 @@ mod tests {
 		let mut queue = VecDeque::new();
 		let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client);
 		let chain_info = client.chain_info();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		let peers = sync.get_lagging_peers(&chain_info, &io);
 		sync.propagate_blocks(&chain_info, &mut io, &[], &peers);
@@ -2076,7 +2346,8 @@ mod tests {
 		// when
 		{
 			let mut queue = VecDeque::new();
-			let mut io = TestIo::new(&mut client, &mut queue, None);
+			let mut ss = TestSnapshotService::new();
+			let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 			io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks);
 			sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]);
 			assert_eq!(io.chain.miner.status().transactions_in_future_queue, 0);
@@ -2090,7 +2361,8 @@ mod tests {
 		}
 		{
 			let mut queue = VecDeque::new();
-			let mut io = TestIo::new(&mut client, &mut queue, None);
+			let mut ss = TestSnapshotService::new();
+			let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 			io.chain.miner.chain_new_blocks(io.chain, &[], &[], &good_blocks, &retracted_blocks);
 			sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks, &[]);
 		}
@@ -2114,7 +2386,8 @@ mod tests {
 		let retracted_blocks = vec![client.block_hash_delta_minus(1)];
 
 		let mut queue = VecDeque::new();
-		let mut io = TestIo::new(&mut client, &mut queue, None);
+		let mut ss = TestSnapshotService::new();
+		let mut io = TestIo::new(&mut client, &mut ss, &mut queue, None);
 
 		// when
 		sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]);

sync/src/lib.rs

@@ -26,40 +26,6 @@
 //! Implements ethereum protocol version 63 as specified here:
 //! https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol
 //!
-//! Usage example:
-//!
-//! ```rust
-//! extern crate ethcore_util as util;
-//! extern crate ethcore_io as io;
-//! extern crate ethcore;
-//! extern crate ethsync;
-//! use std::env;
-//! use io::IoChannel;
-//! use ethcore::client::{Client, ClientConfig};
-//! use ethsync::{EthSync, SyncConfig, ManageNetwork, NetworkConfiguration};
-//! use ethcore::ethereum;
-//! use ethcore::miner::{GasPricer, Miner};
-//!
-//! fn main() {
-//! 	let dir = env::temp_dir();
-//!		let spec = ethereum::new_frontier();
-//! 	let miner = Miner::new(
-//! 		Default::default(),
-//! 		GasPricer::new_fixed(20_000_000_000u64.into()),
-//! 		&spec,
-//! 		None
-//! 	);
-//! 	let client = Client::new(
-//!			ClientConfig::default(),
-//!			&spec,
-//!			&dir,
-//!			miner,
-//!			IoChannel::disconnected()
-//!		).unwrap();
-//! 	let sync = EthSync::new(SyncConfig::default(), client, NetworkConfiguration::from(NetworkConfiguration::new())).unwrap();
-//! 	sync.start_network();
-//! }
-//! ```
 
 extern crate ethcore_network as network;
 extern crate ethcore_io as io;
@@ -83,6 +49,7 @@ extern crate ethcore_ipc as ipc;
 mod chain;
 mod blocks;
 mod sync_io;
+mod snapshot;
 
 #[cfg(test)]
 mod tests;
@@ -96,4 +63,3 @@ pub use api::{EthSync, SyncProvider, SyncClient, NetworkManagerClient, ManageNet
 	ServiceConfiguration, NetworkConfiguration};
 pub use chain::{SyncStatus, SyncState};
 pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError};
-

sync/src/snapshot.rs

@@ -0,0 +1,200 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use util::{H256, Hashable};
+use std::collections::HashSet;
+use ethcore::snapshot::ManifestData;
+
+#[derive(PartialEq, Eq, Debug)]
+pub enum ChunkType {
+	State(H256),
+	Block(H256),
+}
+
+pub struct Snapshot {
+	pending_state_chunks: Vec<H256>,
+	pending_block_chunks: Vec<H256>,
+	downloading_chunks: HashSet<H256>,
+	completed_chunks: HashSet<H256>,
+	snapshot_hash: Option<H256>,
+}
+
+impl Snapshot {
+	/// Create a new instance.
+	pub fn new() -> Snapshot {
+		Snapshot {
+			pending_state_chunks: Vec::new(),
+			pending_block_chunks: Vec::new(),
+			downloading_chunks: HashSet::new(),
+			completed_chunks: HashSet::new(),
+			snapshot_hash: None,
+		}
+	}
+
+	/// Clear everything.
+	pub fn clear(&mut self) {
+		self.pending_state_chunks.clear();
+		self.pending_block_chunks.clear();
+		self.downloading_chunks.clear();
+		self.completed_chunks.clear();
+		self.snapshot_hash = None;
+	}
+
+	/// Reset collection for a manifest RLP
+	pub fn reset_to(&mut self, manifest: &ManifestData, hash: &H256) {
+		self.clear();
+		self.pending_state_chunks = manifest.state_hashes.clone();
+		self.pending_block_chunks = manifest.block_hashes.clone();
+		self.snapshot_hash = Some(hash.clone());
+	}
+
+	/// Validate chunk and mark it as downloaded
+	pub fn validate_chunk(&mut self, chunk: &[u8]) -> Result<ChunkType, ()> {
+		let hash = chunk.sha3();
+		if self.completed_chunks.contains(&hash) {
+			trace!(target: "sync", "Ignored proccessed chunk: {}", hash.hex());
+			return Err(());
+		}
+		self.downloading_chunks.remove(&hash);
+		if self.pending_block_chunks.iter().any(|h| h == &hash) {
+			self.completed_chunks.insert(hash.clone());
+			return Ok(ChunkType::Block(hash));
+		}
+		if self.pending_state_chunks.iter().any(|h| h == &hash) {
+			self.completed_chunks.insert(hash.clone());
+			return Ok(ChunkType::State(hash));
+		}
+		trace!(target: "sync", "Ignored unknown chunk: {}", hash.hex());
+		Err(())
+	}
+
+	/// Find a chunk to download
+	pub fn needed_chunk(&mut self) -> Option<H256> {
+		// check state chunks first
+		let mut chunk = self.pending_state_chunks.iter()
+			.find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h))
+			.cloned();
+		if chunk.is_none() {
+			chunk = self.pending_block_chunks.iter()
+				.find(|&h| !self.downloading_chunks.contains(h) && !self.completed_chunks.contains(h))
+				.cloned();
+		}
+
+		if let Some(hash) = chunk {
+			self.downloading_chunks.insert(hash.clone());
+		}
+		chunk
+	}
+
+	pub fn clear_chunk_download(&mut self, hash: &H256) {
+		self.downloading_chunks.remove(hash);
+	}
+
+	pub fn snapshot_hash(&self) -> Option<H256> {
+		self.snapshot_hash
+	}
+
+	pub fn total_chunks(&self) -> usize {
+		self.pending_block_chunks.len() + self.pending_state_chunks.len()
+	}
+
+	pub fn done_chunks(&self) -> usize {
+		self.total_chunks() - self.completed_chunks.len()
+	}
+
+	pub fn is_complete(&self) -> bool {
+		self.total_chunks() == self.completed_chunks.len()
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use util::*;
+	use super::*;
+	use ethcore::snapshot::ManifestData;
+
+	fn is_empty(snapshot: &Snapshot) -> bool {
+		snapshot.pending_block_chunks.is_empty() &&
+		snapshot.pending_state_chunks.is_empty() &&
+		snapshot.completed_chunks.is_empty() &&
+		snapshot.downloading_chunks.is_empty() &&
+		snapshot.snapshot_hash.is_none()
+	}
+
+	fn test_manifest() -> (ManifestData, H256, Vec<Bytes>, Vec<Bytes>) {
+		let state_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
+		let block_chunks: Vec<Bytes> = (0..20).map(|_| H256::random().to_vec()).collect();
+		let manifest = ManifestData {
+			state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
+			block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
+			state_root: H256::new(),
+			block_number: 42,
+			block_hash: H256::new(),
+		};
+		let mhash = manifest.clone().into_rlp().sha3();
+		(manifest, mhash, state_chunks, block_chunks)
+	}
+
+	#[test]
+	fn create_clear() {
+		let mut snapshot = Snapshot::new();
+		assert!(is_empty(&snapshot));
+		let (manifest, mhash, _, _,) = test_manifest();
+		snapshot.reset_to(&manifest, &mhash);
+		assert!(!is_empty(&snapshot));
+		snapshot.clear();
+		assert!(is_empty(&snapshot));
+	}
+
+	#[test]
+	fn validate_chunks() {
+		let mut snapshot = Snapshot::new();
+		let (manifest, mhash, state_chunks, block_chunks) = test_manifest();
+		snapshot.reset_to(&manifest, &mhash);
+		assert!(snapshot.validate_chunk(&H256::random().to_vec()).is_err());
+
+		let requested: Vec<H256> = (0..40).map(|_| snapshot.needed_chunk().unwrap()).collect();
+		assert!(snapshot.needed_chunk().is_none());
+		assert_eq!(&requested[0..20], &manifest.state_hashes[..]);
+		assert_eq!(&requested[20..40], &manifest.block_hashes[..]);
+		assert_eq!(snapshot.downloading_chunks.len(), 40);
+
+		assert_eq!(snapshot.validate_chunk(&state_chunks[4]), Ok(ChunkType::State(manifest.state_hashes[4].clone())));
+		assert_eq!(snapshot.completed_chunks.len(), 1);
+		assert_eq!(snapshot.downloading_chunks.len(), 39);
+
+		assert_eq!(snapshot.validate_chunk(&block_chunks[10]), Ok(ChunkType::Block(manifest.block_hashes[10].clone())));
+		assert_eq!(snapshot.completed_chunks.len(), 2);
+		assert_eq!(snapshot.downloading_chunks.len(), 38);
+
+		for (i, data) in state_chunks.iter().enumerate() {
+			if i != 4 {
+				assert!(snapshot.validate_chunk(data).is_ok());
+			}
+		}
+
+		for (i, data) in block_chunks.iter().enumerate() {
+			if i != 10 {
+				assert!(snapshot.validate_chunk(data).is_ok());
+			}
+		}
+
+		assert!(snapshot.is_complete());
+		assert_eq!(snapshot.snapshot_hash(), Some(manifest.into_rlp().sha3()));
+	}
+}
+

sync/src/sync_io.rs

@@ -16,6 +16,8 @@
 
 use network::{NetworkContext, PeerId, PacketId, NetworkError};
 use ethcore::client::BlockChainClient;
+use ethcore::snapshot::SnapshotService;
+use api::ETH_PROTOCOL;
 
 /// IO interface for the syning handler.
 /// Provides peer connection management and an interface to the blockchain client.
@@ -31,10 +33,14 @@ pub trait SyncIo {
 	fn send(&mut self, peer_id: PeerId, packet_id: PacketId, data: Vec<u8>) -> Result<(), NetworkError>;
 	/// Get the blockchain
 	fn chain(&self) -> &BlockChainClient;
+	/// Get the snapshot service.
+	fn snapshot_service(&self) -> &SnapshotService;
 	/// Returns peer client identifier string
 	fn peer_info(&self, peer_id: PeerId) -> String {
 		peer_id.to_string()
 	}
+	/// Maximum mutuallt supported ETH protocol version
+	fn eth_protocol_version(&self, peer_id: PeerId) -> u8;
 	/// Returns if the chain block queue empty
 	fn is_chain_queue_empty(&self) -> bool {
 		self.chain().queue_info().is_empty()
@@ -46,15 +52,17 @@ pub trait SyncIo {
 /// Wraps `NetworkContext` and the blockchain client
 pub struct NetSyncIo<'s, 'h> where 'h: 's {
 	network: &'s NetworkContext<'h>,
-	chain: &'s BlockChainClient
+	chain: &'s BlockChainClient,
+	snapshot_service: &'s SnapshotService,
 }
 
 impl<'s, 'h> NetSyncIo<'s, 'h> {
 	/// Creates a new instance from the `NetworkContext` and the blockchain client reference.
-	pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient) -> NetSyncIo<'s, 'h> {
+	pub fn new(network: &'s NetworkContext<'h>, chain: &'s BlockChainClient, snapshot_service: &'s SnapshotService) -> NetSyncIo<'s, 'h> {
 		NetSyncIo {
 			network: network,
 			chain: chain,
+			snapshot_service: snapshot_service,
 		}
 	}
 }
@@ -80,6 +88,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> {
 		self.chain
 	}
 
+	fn snapshot_service(&self) -> &SnapshotService {
+		self.snapshot_service
+	}
+
 	fn peer_info(&self, peer_id: PeerId) -> String {
 		self.network.peer_info(peer_id)
 	}
@@ -87,6 +99,10 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> {
 	fn is_expired(&self) -> bool {
 		self.network.is_expired()
 	}
+
+	fn eth_protocol_version(&self, peer_id: PeerId) -> u8 {
+		self.network.protocol_version(peer_id, ETH_PROTOCOL).unwrap_or(0)
+	}
 }
 
 

sync/src/tests/helpers.rs

@@ -16,22 +16,26 @@
 
 use util::*;
 use network::*;
+use tests::snapshot::*;
 use ethcore::client::{TestBlockChainClient, BlockChainClient};
 use ethcore::header::BlockNumber;
+use ethcore::snapshot::SnapshotService;
 use sync_io::SyncIo;
 use chain::ChainSync;
 use ::SyncConfig;
 
 pub struct TestIo<'p> {
 	pub chain: &'p mut TestBlockChainClient,
+	pub snapshot_service: &'p TestSnapshotService,
 	pub queue: &'p mut VecDeque<TestPacket>,
 	pub sender: Option<PeerId>,
 }
 
 impl<'p> TestIo<'p> {
-	pub fn new(chain: &'p mut TestBlockChainClient, queue: &'p mut VecDeque<TestPacket>, sender: Option<PeerId>) -> TestIo<'p> {
+	pub fn new(chain: &'p mut TestBlockChainClient, ss: &'p TestSnapshotService, queue: &'p mut VecDeque<TestPacket>, sender: Option<PeerId>) -> TestIo<'p> {
 		TestIo {
 			chain: chain,
+			snapshot_service: ss,
 			queue: queue,
 			sender: sender
 		}
@@ -70,6 +74,14 @@ impl<'p> SyncIo for TestIo<'p> {
 	fn chain(&self) -> &BlockChainClient {
 		self.chain
 	}
+
+	fn snapshot_service(&self) -> &SnapshotService {
+		self.snapshot_service
+	}
+
+	fn eth_protocol_version(&self, _peer: PeerId) -> u8 {
+		64
+	}
 }
 
 pub struct TestPacket {
@@ -80,6 +92,7 @@ pub struct TestPacket {
 
 pub struct TestPeer {
 	pub chain: TestBlockChainClient,
+	pub snapshot_service: Arc<TestSnapshotService>,
 	pub sync: RwLock<ChainSync>,
 	pub queue: VecDeque<TestPacket>,
 }
@@ -103,9 +116,11 @@ impl TestNet {
 			let chain = TestBlockChainClient::new();
 			let mut config = SyncConfig::default();
 			config.fork_block = fork;
+			let ss = Arc::new(TestSnapshotService::new());
 			let sync = ChainSync::new(config, &chain);
 			net.peers.push(TestPeer {
 				sync: RwLock::new(sync),
+				snapshot_service: ss,
 				chain: chain,
 				queue: VecDeque::new(),
 			});
@@ -126,7 +141,7 @@ impl TestNet {
 			for client in 0..self.peers.len() {
 				if peer != client {
 					let mut p = self.peers.get_mut(peer).unwrap();
-					p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &mut p.queue, Some(client as PeerId)), client as PeerId);
+					p.sync.write().on_peer_connected(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(client as PeerId)), client as PeerId);
 				}
 			}
 		}
@@ -137,22 +152,22 @@ impl TestNet {
 			if let Some(packet) = self.peers[peer].queue.pop_front() {
 				let mut p = self.peers.get_mut(packet.recipient).unwrap();
 				trace!("--- {} -> {} ---", peer, packet.recipient);
-				ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data);
+				ChainSync::dispatch_packet(&p.sync, &mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, Some(peer as PeerId)), peer as PeerId, packet.packet_id, &packet.data);
 				trace!("----------------");
 			}
 			let mut p = self.peers.get_mut(peer).unwrap();
-			p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &mut p.queue, None));
+			p.sync.write().maintain_sync(&mut TestIo::new(&mut p.chain, &p.snapshot_service, &mut p.queue, None));
 		}
 	}
 
 	pub fn sync_step_peer(&mut self, peer_num: usize) {
 		let mut peer = self.peer_mut(peer_num);
-		peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None));
+		peer.sync.write().maintain_sync(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None));
 	}
 
 	pub fn restart_peer(&mut self, i: usize) {
 		let peer = self.peer_mut(i);
-		peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None));
+		peer.sync.write().restart(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None));
 	}
 
 	pub fn sync(&mut self) -> u32 {
@@ -181,6 +196,6 @@ impl TestNet {
 
 	pub fn trigger_chain_new_blocks(&mut self, peer_id: usize) {
 		let mut peer = self.peer_mut(peer_id);
-		peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None), &[], &[], &[], &[], &[]);
+		peer.sync.write().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &peer.snapshot_service, &mut peer.queue, None), &[], &[], &[], &[], &[]);
 	}
 }

sync/src/tests/mod.rs

@@ -15,5 +15,6 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 pub mod helpers;
+pub mod snapshot;
 mod chain;
 mod rpc;

sync/src/tests/snapshot.rs

@@ -0,0 +1,123 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use util::*;
+use ethcore::snapshot::{SnapshotService, ManifestData, RestorationStatus};
+use ethcore::header::BlockNumber;
+use ethcore::client::{EachBlockWith};
+use super::helpers::*;
+
+pub struct TestSnapshotService {
+	manifest: Option<ManifestData>,
+	chunks: HashMap<H256, Bytes>,
+
+	restoration_manifest: Mutex<Option<ManifestData>>,
+	state_restoration_chunks: Mutex<HashMap<H256, Bytes>>,
+	block_restoration_chunks: Mutex<HashMap<H256, Bytes>>,
+}
+
+impl TestSnapshotService {
+	pub fn new() -> TestSnapshotService {
+		TestSnapshotService {
+			manifest: None,
+			chunks: HashMap::new(),
+			restoration_manifest: Mutex::new(None),
+			state_restoration_chunks: Mutex::new(HashMap::new()),
+			block_restoration_chunks: Mutex::new(HashMap::new()),
+		}
+	}
+
+	pub fn new_with_snapshot(num_chunks: usize, block_hash: H256, block_number: BlockNumber) -> TestSnapshotService {
+		let num_state_chunks = num_chunks / 2;
+		let num_block_chunks = num_chunks - num_state_chunks;
+		let state_chunks: Vec<Bytes> = (0..num_state_chunks).map(|_| H256::random().to_vec()).collect();
+		let block_chunks: Vec<Bytes> = (0..num_block_chunks).map(|_| H256::random().to_vec()).collect();
+		let manifest = ManifestData {
+			state_hashes: state_chunks.iter().map(|data| data.sha3()).collect(),
+			block_hashes: block_chunks.iter().map(|data| data.sha3()).collect(),
+			state_root: H256::new(),
+			block_number: block_number,
+			block_hash: block_hash,
+		};
+		let mut chunks: HashMap<H256, Bytes> = state_chunks.into_iter().map(|data| (data.sha3(), data)).collect();
+		chunks.extend(block_chunks.into_iter().map(|data| (data.sha3(), data)));
+		TestSnapshotService {
+			manifest: Some(manifest),
+			chunks: chunks,
+			restoration_manifest: Mutex::new(None),
+			state_restoration_chunks: Mutex::new(HashMap::new()),
+			block_restoration_chunks: Mutex::new(HashMap::new()),
+		}
+	}
+}
+
+impl SnapshotService for TestSnapshotService {
+	fn manifest(&self) -> Option<ManifestData> {
+		self.manifest.as_ref().cloned()
+	}
+
+	fn chunk(&self, hash: H256) -> Option<Bytes> {
+		self.chunks.get(&hash).cloned()
+	}
+
+	fn status(&self) -> RestorationStatus {
+		match &*self.restoration_manifest.lock() {
+			&Some(ref manifest) if self.state_restoration_chunks.lock().len() == manifest.state_hashes.len() &&
+				self.block_restoration_chunks.lock().len() == manifest.block_hashes.len() => RestorationStatus::Inactive,
+			&Some(_) => RestorationStatus::Ongoing {
+				state_chunks_done: self.state_restoration_chunks.lock().len() as u32,
+				block_chunks_done: self.block_restoration_chunks.lock().len() as u32,
+			},
+			&None => RestorationStatus::Inactive,
+		}
+	}
+
+	fn begin_restore(&self, manifest: ManifestData) {
+		*self.restoration_manifest.lock() = Some(manifest);
+		self.state_restoration_chunks.lock().clear();
+		self.block_restoration_chunks.lock().clear();
+	}
+
+	fn abort_restore(&self) {
+		*self.restoration_manifest.lock() = None;
+		self.state_restoration_chunks.lock().clear();
+		self.block_restoration_chunks.lock().clear();
+	}
+
+	fn restore_state_chunk(&self, hash: H256, chunk: Bytes) {
+		if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.state_hashes.iter().any(|h| h == &hash)) {
+			self.state_restoration_chunks.lock().insert(hash, chunk);
+		}
+	}
+
+	fn restore_block_chunk(&self, hash: H256, chunk: Bytes) {
+		if self.restoration_manifest.lock().as_ref().map_or(false, |ref m| m.block_hashes.iter().any(|h| h == &hash)) {
+			self.block_restoration_chunks.lock().insert(hash, chunk);
+		}
+	}
+}
+
+#[test]
+fn snapshot_sync() {
+	::env_logger::init().ok();
+	let mut net = TestNet::new(2);
+	net.peer_mut(0).snapshot_service = Arc::new(TestSnapshotService::new_with_snapshot(16, H256::new(), 1));
+	net.peer_mut(0).chain.add_blocks(1, EachBlockWith::Nothing);
+	net.sync_steps(19); // status + manifest + chunks
+	assert_eq!(net.peer(1).snapshot_service.state_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().state_hashes.len());
+	assert_eq!(net.peer(1).snapshot_service.block_restoration_chunks.lock().len(), net.peer(0).snapshot_service.manifest.as_ref().unwrap().block_hashes.len());
+}
+

util/network/src/host.rs

@@ -282,6 +282,12 @@ impl<'s> NetworkContext<'s> {
 		}
 		"unknown".to_owned()
 	}
+
+	/// Returns max version for a given protocol.
+	pub fn protocol_version(&self, peer: PeerId, protocol: &str) -> Option<u8> {
+		let session = self.resolve_session(peer);
+		session.and_then(|s| s.lock().capability_version(protocol))
+	}
 }
 
 /// Shared host information

util/network/src/session.rs

@@ -243,6 +243,11 @@ impl Session {
 		self.info.capabilities.iter().any(|c| c.protocol == protocol)
 	}
 
+	/// Checks if peer supports given capability
+	pub fn capability_version(&self, protocol: &str) -> Option<u8> {
+		self.info.capabilities.iter().filter_map(|c| if c.protocol == protocol { Some(c.version) } else { None }).max()
+	}
+
 	/// Register the session socket with the event loop
 	pub fn register_socket<Host:Handler<Timeout = Token>>(&self, reg: Token, event_loop: &mut EventLoop<Host>) -> Result<(), NetworkError> {
 		if self.expired() {

util/src/kvdb.rs

@@ -16,9 +16,11 @@
 
 //! Key-Value store abstraction with `RocksDB` backend.
 
+use std::io::ErrorKind;
 use common::*;
 use elastic_array::*;
 use std::default::Default;
+use std::path::PathBuf;
 use rlp::{UntrustedRlp, RlpType, View, Compressible};
 use rocksdb::{DB, Writable, WriteBatch, WriteOptions, IteratorMode, DBIterator,
 	Options, DBCompactionStyle, BlockBasedOptions, Direction, Cache, Column};
@@ -189,12 +191,18 @@ impl<'a> Iterator for DatabaseIterator {
 	}
 }
 
+struct DBAndColumns {
+	db: DB,
+	cfs: Vec<Column>,
+}
+
 /// Key-Value database.
 pub struct Database {
-	db: DB,
+	db: RwLock<Option<DBAndColumns>>,
+	config: DatabaseConfig,
 	write_opts: WriteOptions,
-	cfs: Vec<Column>,
 	overlay: RwLock<Vec<HashMap<ElasticArray32<u8>, KeyState>>>,
+	path: String,
 }
 
 impl Database {
@@ -278,11 +286,13 @@ impl Database {
 			},
 			Err(s) => { return Err(s); }
 		};
+		let num_cols = cfs.len();
 		Ok(Database {
-			db: db,
+			db: RwLock::new(Some(DBAndColumns{ db: db, cfs: cfs })),
+			config: config.clone(),
 			write_opts: write_opts,
-			overlay: RwLock::new((0..(cfs.len() + 1)).map(|_| HashMap::new()).collect()),
+			overlay: RwLock::new((0..(num_cols + 1)).map(|_| HashMap::new()).collect()),
-			cfs: cfs,
+			path: path.to_owned(),
 		})
 	}
 
@@ -320,94 +330,167 @@ impl Database {
 
 	/// Commit buffered changes to database.
 	pub fn flush(&self) -> Result<(), String> {
-		let batch = WriteBatch::new();
+		match &*self.db.read() {
-		let mut overlay = self.overlay.write();
+			&Some(DBAndColumns { ref db, ref cfs }) => {
+				let batch = WriteBatch::new();
+				let mut overlay = self.overlay.write();
 
-		for (c, column) in overlay.iter_mut().enumerate() {
+				for (c, column) in overlay.iter_mut().enumerate() {
-			let column_data = mem::replace(column, HashMap::new());
+					let column_data = mem::replace(column, HashMap::new());
-			for (key, state) in column_data.into_iter() {
+					for (key, state) in column_data.into_iter() {
-				match state {
+						match state {
-					KeyState::Delete => {
+							KeyState::Delete => {
-						if c > 0 {
+								if c > 0 {
-							try!(batch.delete_cf(self.cfs[c - 1], &key));
+									try!(batch.delete_cf(cfs[c - 1], &key));
-						} else {
+								} else {
-							try!(batch.delete(&key));
+									try!(batch.delete(&key));
-						}
+								}
-					},
+							},
-					KeyState::Insert(value) => {
+							KeyState::Insert(value) => {
-						if c > 0 {
+								if c > 0 {
-							try!(batch.put_cf(self.cfs[c - 1], &key, &value));
+									try!(batch.put_cf(cfs[c - 1], &key, &value));
-						} else {
+								} else {
-							try!(batch.put(&key, &value));
+									try!(batch.put(&key, &value));
-						}
+								}
-					},
+							},
-					KeyState::InsertCompressed(value) => {
+							KeyState::InsertCompressed(value) => {
-						let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
+								let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-						if c > 0 {
+								if c > 0 {
-							try!(batch.put_cf(self.cfs[c - 1], &key, &compressed));
+									try!(batch.put_cf(cfs[c - 1], &key, &compressed));
-						} else {
+								} else {
-							try!(batch.put(&key, &value));
+									try!(batch.put(&key, &value));
+								}
+							}
 						}
 					}
 				}
-			}
+				db.write_opt(batch, &self.write_opts)
+			},
+			&None => Err("Database is closed".to_owned())
 		}
-		self.db.write_opt(batch, &self.write_opts)
 	}
 
 
 	/// Commit transaction to database.
 	pub fn write(&self, tr: DBTransaction) -> Result<(), String> {
-		let batch = WriteBatch::new();
+		match &*self.db.read() {
-		let ops = tr.ops;
+			&Some(DBAndColumns { ref db, ref cfs }) => {
-		for op in ops {
+				let batch = WriteBatch::new();
-			match op {
+				let ops = tr.ops;
-				DBOp::Insert { col, key, value } => {
+				for op in ops {
-					try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(self.cfs[c as usize], &key, &value)))
+					match op {
-				},
+						DBOp::Insert { col, key, value } => {
-				DBOp::InsertCompressed { col, key, value } => {
+							try!(col.map_or_else(|| batch.put(&key, &value), |c| batch.put_cf(cfs[c as usize], &key, &value)))
-					let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
+						},
-					try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(self.cfs[c as usize], &key, &compressed)))
+						DBOp::InsertCompressed { col, key, value } => {
-				},
+							let compressed = UntrustedRlp::new(&value).compress(RlpType::Blocks);
-				DBOp::Delete { col, key } => {
+							try!(col.map_or_else(|| batch.put(&key, &compressed), |c| batch.put_cf(cfs[c as usize], &key, &compressed)))
-					try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(self.cfs[c as usize], &key)))
+						},
-				},
+						DBOp::Delete { col, key } => {
-			}
+							try!(col.map_or_else(|| batch.delete(&key), |c| batch.delete_cf(cfs[c as usize], &key)))
+						},
+					}
+				}
+				db.write_opt(batch, &self.write_opts)
+			},
+			&None => Err("Database is closed".to_owned())
 		}
-		self.db.write_opt(batch, &self.write_opts)
 	}
 
 	/// Get value by key.
 	pub fn get(&self, col: Option<u32>, key: &[u8]) -> Result<Option<Bytes>, String> {
-		let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
+		match &*self.db.read() {
-		match overlay.get(key) {
+			&Some(DBAndColumns { ref db, ref cfs }) => {
-			Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
+				let overlay = &self.overlay.read()[Self::to_overlay_column(col)];
-			Some(&KeyState::Delete) => Ok(None),
+				match overlay.get(key) {
-			None => {
+					Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => Ok(Some(value.clone())),
-				col.map_or_else(
+					Some(&KeyState::Delete) => Ok(None),
-					|| self.db.get(key).map(|r| r.map(|v| v.to_vec())),
+					None => {
-					|c| self.db.get_cf(self.cfs[c as usize], key).map(|r| r.map(|v| v.to_vec())))
+						col.map_or_else(
+							|| db.get(key).map(|r| r.map(|v| v.to_vec())),
+							|c| db.get_cf(cfs[c as usize], key).map(|r| r.map(|v| v.to_vec())))
+					},
+				}
 			},
+			&None => Ok(None),
 		}
 	}
 
 	/// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values.
-	// TODO: support prefix seek for unflushed ata
+	// TODO: support prefix seek for unflushed data
 	pub fn get_by_prefix(&self, col: Option<u32>, prefix: &[u8]) -> Option<Box<[u8]>> {
-		let mut iter = col.map_or_else(|| self.db.iterator(IteratorMode::From(prefix, Direction::Forward)),
+		match &*self.db.read() {
-			|c| self.db.iterator_cf(self.cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
+			&Some(DBAndColumns { ref db, ref cfs }) => {
-		match iter.next() {
+				let mut iter = col.map_or_else(|| db.iterator(IteratorMode::From(prefix, Direction::Forward)),
-			// TODO: use prefix_same_as_start read option (not availabele in C API currently)
+					|c| db.iterator_cf(cfs[c as usize], IteratorMode::From(prefix, Direction::Forward)).unwrap());
-			Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
+				match iter.next() {
-			_ => None
+					// TODO: use prefix_same_as_start read option (not availabele in C API currently)
+					Some((k, v)) => if k[0 .. prefix.len()] == prefix[..] { Some(v) } else { None },
+					_ => None
+				}
+			},
+			&None => None,
 		}
 	}
 
 	/// Get database iterator for flushed data.
 	pub fn iter(&self, col: Option<u32>) -> DatabaseIterator {
 		//TODO: iterate over overlay
-		col.map_or_else(|| DatabaseIterator { iter: self.db.iterator(IteratorMode::Start) },
+		match &*self.db.read() {
-			|c| DatabaseIterator { iter: self.db.iterator_cf(self.cfs[c as usize], IteratorMode::Start).unwrap() })
+			&Some(DBAndColumns { ref db, ref cfs }) => {
+				col.map_or_else(|| DatabaseIterator { iter: db.iterator(IteratorMode::Start) },
+					|c| DatabaseIterator { iter: db.iterator_cf(cfs[c as usize], IteratorMode::Start).unwrap() })
+			},
+			&None => panic!("Not supported yet") //TODO: return an empty iterator or change return type
+		}
+	}
+
+	/// Close the database
+	fn close(&self) {
+		*self.db.write() = None;
+		self.overlay.write().clear();
+	}
+
+	/// Restore the database from a copy at given path.
+	pub fn restore(&self, new_db: &str) -> Result<(), UtilError> {
+		self.close();
+
+		let mut backup_db = PathBuf::from(&self.path);
+		backup_db.pop();
+		backup_db.push("backup_db");
+		println!("Path at {:?}", self.path);
+		println!("Backup at {:?}", backup_db);
+
+		let existed = match fs::rename(&self.path, &backup_db) {
+			Ok(_) => true,
+			Err(e) => if let ErrorKind::NotFound = e.kind() {
+				false
+			} else {
+				return Err(e.into());
+			}
+		};
+
+		match fs::rename(&new_db, &self.path) {
+			Ok(_) => {
+				// clean up the backup.
+				if existed {
+					try!(fs::remove_dir_all(&backup_db));
+				}
+			}
+			Err(e) => {
+				// restore the backup.
+				if existed {
+					try!(fs::rename(&backup_db, &self.path));
+				}
+				return Err(e.into())
+			}
+		}
+
+		// reopen the database and steal handles into self
+		let db = try!(Self::open(&self.config, &self.path));
+		*self.db.write() = mem::replace(&mut *db.db.write(), None);
+		*self.overlay.write() = mem::replace(&mut *db.overlay.write(), Vec::new());
+		Ok(())
 	}
 }
 

util/src/misc.rs

@@ -16,7 +16,6 @@
 
 //! Diff misc.
 
-use std::fs::File;
 use common::*;
 use rlp::{Stream, RlpStream};
 use target_info::Target;
@@ -33,14 +32,6 @@ pub enum Filth {
 	Dirty,
 }
 
-/// Read the whole contents of a file `name`.
-pub fn contents(name: &str) -> Result<Bytes, UtilError> {
-	let mut file = try!(File::open(name));
-	let mut ret: Vec<u8> = Vec::new();
-	try!(file.read_to_end(&mut ret));
-	Ok(ret)
-}
-
 /// Get the standard version string for this software.
 pub fn version() -> String {
 	let sha3 = short_sha();