Merge pull request #3543 from ethcore/hash-fetch

Moving fetching of hash-addressed dapps/content to separate crate.

Cargo.lock

@@ -11,6 +11,7 @@ dependencies = [
  "ethcore 1.5.0",
  "ethcore-dapps 1.5.0",
  "ethcore-devtools 1.4.0",
+ "ethcore-hash-fetch 1.5.0",
  "ethcore-io 1.5.0",
  "ethcore-ipc 1.4.0",
  "ethcore-ipc-codegen 1.4.0",
@@ -334,8 +335,8 @@ version = "1.5.0"
 dependencies = [
  "clippy 0.0.96 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethcore-devtools 1.4.0",
+ "ethcore-hash-fetch 1.5.0",
  "ethcore-rpc 1.5.0",
  "ethcore-util 1.5.0",
  "fetch 0.1.0",
@@ -366,6 +367,18 @@ dependencies = [
  "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ethcore-hash-fetch"
+version = "1.5.0"
+dependencies = [
+ "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ethcore-util 1.5.0",
+ "fetch 0.1.0",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "ethcore-io"
 version = "1.5.0"

Cargo.toml

@@ -43,6 +43,7 @@ ethcore-ipc-nano = { path = "ipc/nano" }
 ethcore-ipc = { path = "ipc/rpc" }
 ethcore-ipc-hypervisor = { path = "ipc/hypervisor" }
 ethcore-logger = { path = "logger" }
+ethcore-hash-fetch = { path = "ethcore/hash-fetch" }
 rlp = { path = "util/rlp" }
 ethcore-stratum = { path = "stratum" }
 ethcore-dapps = { path = "dapps", optional = true }

dapps/Cargo.toml

@@ -20,20 +20,20 @@ url = "1.0"
 rustc-serialize = "0.3"
 serde = "0.8"
 serde_json = "0.8"
-ethabi = "0.2.2"
 linked-hash-map = "0.3"
 parity-dapps-glue = "1.4"
 mime = "0.2"
+mime_guess = "1.6.1"
 time = "0.1.35"
 serde_macros = { version = "0.8", optional = true }
 zip = { version = "0.1", default-features = false }
 ethcore-devtools = { path = "../devtools" }
 ethcore-rpc = { path = "../rpc" }
 ethcore-util = { path = "../util" }
+ethcore-hash-fetch = { path = "../ethcore/hash-fetch" }
 fetch = { path = "../util/fetch" }
 parity-ui = { path = "./ui" }
 
-mime_guess = { version = "1.6.1" }
 clippy = { version = "0.0.96", optional = true}
 
 [build-dependencies]

dapps/src/apps/fetcher.rs

@@ -24,6 +24,7 @@ use std::io::{self, Read, Write};
 use std::path::PathBuf;
 use std::sync::Arc;
 use rustc_serialize::hex::FromHex;
+use hash_fetch::urlhint::{URLHintContract, URLHint, URLHintResult};
 
 use hyper;
 use hyper::status::StatusCode;
@@ -37,7 +38,6 @@ use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
 use endpoint::{Endpoint, EndpointPath, Handler};
 use apps::cache::{ContentCache, ContentStatus};
 use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
-use apps::urlhint::{URLHintContract, URLHint, URLHintResult};
 
 /// Limit of cached dapps/content
 const MAX_CACHED_DAPPS: usize = 20;
@@ -402,10 +402,11 @@ mod tests {
 	use std::env;
 	use std::sync::Arc;
 	use util::Bytes;
+	use hash_fetch::urlhint::{URLHint, URLHintResult};
+
+	use apps::cache::ContentStatus;
 	use endpoint::EndpointInfo;
 	use page::LocalPageEndpoint;
-	use apps::cache::ContentStatus;
-	use apps::urlhint::{URLHint, URLHintResult};
 	use super::ContentFetcher;
 
 	struct FakeResolver;

dapps/src/apps/mod.rs

@@ -21,7 +21,6 @@ use parity_dapps::WebApp;
 
 mod cache;
 mod fs;
-pub mod urlhint;
 pub mod fetcher;
 pub mod manifest;
 

dapps/src/lib.rs

@@ -51,13 +51,13 @@ extern crate serde;
 extern crate serde_json;
 extern crate zip;
 extern crate rand;
-extern crate ethabi;
 extern crate jsonrpc_core;
 extern crate jsonrpc_http_server;
 extern crate mime_guess;
 extern crate rustc_serialize;
 extern crate ethcore_rpc;
 extern crate ethcore_util as util;
+extern crate ethcore_hash_fetch as hash_fetch;
 extern crate linked_hash_map;
 extern crate fetch;
 extern crate parity_dapps_glue as parity_dapps;
@@ -84,12 +84,11 @@ mod url;
 #[cfg(test)]
 mod tests;
 
-pub use self::apps::urlhint::ContractClient;
-
 use std::sync::{Arc, Mutex};
 use std::net::SocketAddr;
 use std::collections::HashMap;
 
+use hash_fetch::urlhint::ContractClient;
 use jsonrpc_core::{IoHandler, IoDelegate};
 use router::auth::{Authorization, NoAuth, HttpBasicAuth};
 use ethcore_rpc::Extendable;
@@ -219,7 +218,7 @@ impl Server {
 	) -> Result<Server, ServerError> {
 		let panic_handler = Arc::new(Mutex::new(None));
 		let authorization = Arc::new(authorization);
-		let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status, signer_address.clone()));
+		let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(hash_fetch::urlhint::URLHintContract::new(registrar), sync_status, signer_address.clone()));
 		let endpoints = Arc::new(apps::all_endpoints(dapps_path, signer_address.clone()));
 		let cors_domains = Self::cors_domains(signer_address.clone());
 

dapps/src/tests/helpers.rs

@@ -22,7 +22,7 @@ use env_logger::LogBuilder;
 
 use ServerBuilder;
 use Server;
-use apps::urlhint::ContractClient;
+use hash_fetch::urlhint::ContractClient;
 use util::{Bytes, Address, Mutex, ToPretty};
 use devtools::http_client;
 

ethcore/hash-fetch/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+description = "Fetching hash-addressed content."
+homepage = "https://ethcore.io"
+license = "GPL-3.0"
+name = "ethcore-hash-fetch"
+version = "1.5.0"
+authors = ["Ethcore <admin@ethcore.io>"]
+
+[dependencies]
+log = "0.3"
+rustc-serialize = "0.3"
+ethabi = "0.2.2"
+mime_guess = "1.6.1"
+fetch = { path = "../../util/fetch" }
+ethcore-util = { path = "../../util" }

ethcore/hash-fetch/src/client.rs

@@ -0,0 +1,114 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hash-addressed content resolver & fetcher.
+
+use std::{io, fs};
+use std::sync::Arc;
+use std::path::PathBuf;
+
+use util::{Mutex, H256, sha3};
+use fetch::{Fetch, FetchError, Client as FetchClient};
+
+use urlhint::{ContractClient, URLHintContract, URLHint, URLHintResult};
+
+/// API for fetching by hash.
+pub trait HashFetch {
+	/// Fetch hash-addressed content.
+	/// Parameters:
+	/// 1. `hash` - content hash
+	/// 2. `on_done` - callback function invoked when the content is ready (or there was error during fetch)
+	///
+	/// This function may fail immediately when fetch cannot be initialized or content cannot be resolved.
+	fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>) -> Result<(), Error>;
+}
+
+/// Hash-fetching error.
+#[derive(Debug)]
+pub enum Error {
+	/// Hash could not be resolved to a valid content address.
+	NoResolution,
+	/// Downloaded content hash does not match.
+	HashMismatch { expected: H256, got: H256 },
+	/// IO Error while validating hash.
+	IO(io::Error),
+	/// Error during fetch.
+	Fetch(FetchError),
+}
+
+impl From<FetchError> for Error {
+	fn from(error: FetchError) -> Self {
+		Error::Fetch(error)
+	}
+}
+
+impl From<io::Error> for Error {
+	fn from(error: io::Error) -> Self {
+		Error::IO(error)
+	}
+}
+
+/// Default Hash-fetching client using on-chain contract to resolve hashes to URLs.
+pub struct Client {
+	contract: URLHintContract,
+	fetch: Mutex<FetchClient>,
+}
+
+impl Client {
+	/// Creates new instance of the `Client` given on-chain contract client.
+	pub fn new(contract: Arc<ContractClient>) -> Self {
+		Client {
+			contract: URLHintContract::new(contract),
+			fetch: Mutex::new(FetchClient::default()),
+		}
+	}
+}
+
+impl HashFetch for Client {
+	fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>) -> Result<(), Error> {
+		debug!(target: "dapps", "Fetching: {:?}", hash);
+
+		let url = try!(
+			self.contract.resolve(hash.to_vec()).map(|content| match content {
+				URLHintResult::Dapp(dapp) => {
+					dapp.url()
+				},
+				URLHintResult::Content(content) => {
+					content.url
+				},
+			}).ok_or_else(|| Error::NoResolution)
+		);
+
+		debug!(target: "dapps", "Resolved {:?} to {:?}. Fetching...", hash, url);
+
+		self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| {
+			fn validate_hash(hash: H256, result: Result<PathBuf, FetchError>) -> Result<PathBuf, Error> {
+				let path = try!(result);
+				let mut file_reader = io::BufReader::new(try!(fs::File::open(&path)));
+				let content_hash = try!(sha3(&mut file_reader));
+
+				if content_hash != hash {
+					Err(Error::HashMismatch{ got: content_hash, expected: hash })
+				} else {
+					Ok(path)
+				}
+			}
+
+			debug!(target: "dapps", "Content fetched, validating hash ({:?})", hash);
+			on_done(validate_hash(hash, result))
+		})).map_err(Into::into)
+	}
+}

ethcore/hash-fetch/src/lib.rs

@@ -0,0 +1,33 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hash-addressed content resolver & fetcher.
+
+#![warn(missing_docs)]
+
+#[macro_use]
+extern crate log;
+extern crate rustc_serialize;
+extern crate mime_guess;
+extern crate ethabi;
+extern crate ethcore_util as util;
+extern crate fetch;
+
+mod client;
+
+pub mod urlhint;
+
+pub use client::{HashFetch, Client};

ethcore/hash-fetch/src/urlhint.rs

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
+//! URLHint Contract
+
 use std::fmt;
 use std::sync::Arc;
 use rustc_serialize::hex::ToHex;
@@ -24,15 +26,30 @@ use util::{Address, Bytes, Hashable};
 
 const COMMIT_LEN: usize = 20;
 
+/// RAW Contract interface.
+/// Should execute transaction using current blockchain state.
+pub trait ContractClient: Send + Sync {
+	/// Get registrar address
+	fn registrar(&self) -> Result<Address, String>;
+	/// Call Contract
+	fn call(&self, address: Address, data: Bytes) -> Result<Bytes, String>;
+}
+
+/// Github-hosted dapp.
 #[derive(Debug, PartialEq)]
 pub struct GithubApp {
+	/// Github Account
 	pub account: String,
+	/// Github Repository
 	pub repo: String,
+	/// Commit on Github
 	pub commit: [u8;COMMIT_LEN],
+	/// Dapp owner address
 	pub owner: Address,
 }
 
 impl GithubApp {
+	/// Returns URL of this Github-hosted dapp package.
 	pub fn url(&self) -> String {
 		// Since https fetcher doesn't support redirections we use direct link
 		// format!("https://github.com/{}/{}/archive/{}.zip", self.account, self.repo, self.commit.to_hex())
@@ -53,22 +70,17 @@ impl GithubApp {
 	}
 }
 
+/// Hash-Addressed Content
 #[derive(Debug, PartialEq)]
 pub struct Content {
+	/// URL of the content
 	pub url: String,
+	/// MIME type of the content
 	pub mime: String,
+	/// Content owner address
 	pub owner: Address,
 }
 
-/// RAW Contract interface.
-/// Should execute transaction using current blockchain state.
-pub trait ContractClient: Send + Sync {
-	/// Get registrar address
-	fn registrar(&self) -> Result<Address, String>;
-	/// Call Contract
-	fn call(&self, address: Address, data: Bytes) -> Result<Bytes, String>;
-}
-
 /// Result of resolving id to URL
 #[derive(Debug, PartialEq)]
 pub enum URLHintResult {
@@ -84,6 +96,7 @@ pub trait URLHint {
 	fn resolve(&self, id: Bytes) -> Option<URLHintResult>;
 }
 
+/// `URLHintContract` API
 pub struct URLHintContract {
 	urlhint: Contract,
 	registrar: Contract,
@@ -91,9 +104,10 @@ pub struct URLHintContract {
 }
 
 impl URLHintContract {
+	/// Creates new `URLHintContract`
 	pub fn new(client: Arc<ContractClient>) -> Self {
-		let urlhint = Interface::load(include_bytes!("./urlhint.json")).expect("urlhint.json is valid ABI");
+		let urlhint = Interface::load(include_bytes!("../res/urlhint.json")).expect("urlhint.json is valid ABI");
-		let registrar = Interface::load(include_bytes!("./registrar.json")).expect("registrar.json is valid ABI");
+		let registrar = Interface::load(include_bytes!("../res/registrar.json")).expect("registrar.json is valid ABI");
 
 		URLHintContract {
 			urlhint: Contract::new(urlhint),
@@ -244,11 +258,6 @@ fn guess_mime_type(url: &str) -> Option<String> {
 	})
 }
 
-#[cfg(test)]
-pub fn test_guess_mime_type(url: &str) -> Option<String> {
-	guess_mime_type(url)
-}
-
 fn as_string<T: fmt::Debug>(e: T) -> String {
 	format!("{:?}", e)
 }
@@ -260,6 +269,7 @@ mod tests {
 	use rustc_serialize::hex::FromHex;
 
 	use super::*;
+	use super::guess_mime_type;
 	use util::{Bytes, Address, Mutex, ToPretty};
 
 	struct FakeRegistrar {
@@ -390,10 +400,10 @@ mod tests {
 		let url5 = "https://ethcore.io/parity.png";
 
 
-		assert_eq!(test_guess_mime_type(url1), None);
+		assert_eq!(guess_mime_type(url1), None);
-		assert_eq!(test_guess_mime_type(url2), Some("image/png".into()));
+		assert_eq!(guess_mime_type(url2), Some("image/png".into()));
-		assert_eq!(test_guess_mime_type(url3), Some("image/png".into()));
+		assert_eq!(guess_mime_type(url3), Some("image/png".into()));
-		assert_eq!(test_guess_mime_type(url4), Some("image/jpeg".into()));
+		assert_eq!(guess_mime_type(url4), Some("image/jpeg".into()));
-		assert_eq!(test_guess_mime_type(url5), Some("image/png".into()));
+		assert_eq!(guess_mime_type(url5), Some("image/png".into()));
 	}
 }

parity/dapps.rs

@@ -110,7 +110,7 @@ mod server {
 
 	use rpc_apis;
 	use ethcore_rpc::is_major_importing;
-	use ethcore_dapps::ContractClient;
+	use hash_fetch::urlhint::ContractClient;
 
 	pub use ethcore_dapps::Server as WebappServer;
 

parity/main.rs

@@ -42,6 +42,7 @@ extern crate ethcore_ipc_nano as nanoipc;
 extern crate serde;
 extern crate serde_json;
 extern crate rlp;
+extern crate ethcore_hash_fetch as hash_fetch;
 
 extern crate json_ipc_server as jsonipc;
 

util/bigint/src/uint.rs

@@ -683,6 +683,7 @@ macro_rules! construct_uint {
  					bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8;
 				}
 			}
+
 			#[inline]
 			fn exp10(n: usize) -> Self {
 				match n {