Hash Content RPC method (#2355)

* Moving file fetching to separate crate.

* ethcore_hashContent

* Tests running on mocked fetch.

* Limiting size of downloadable assets

Cargo.lock

@@ -317,7 +317,7 @@ dependencies = [
  "ethcore-devtools 1.4.0",
  "ethcore-rpc 1.4.0",
  "ethcore-util 1.4.0",
- "https-fetch 0.1.0",
+ "fetch 0.1.0",
  "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
  "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)",
@@ -466,6 +466,7 @@ dependencies = [
  "ethkey 0.2.0",
  "ethstore 0.1.0",
  "ethsync 1.4.0",
+ "fetch 0.1.0",
  "json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)",
  "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)",
@@ -639,6 +640,16 @@ dependencies = [
  "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "fetch"
+version = "0.1.0"
+dependencies = [
+ "https-fetch 0.1.0",
+ "hyper 0.9.4 (git+https://github.com/ethcore/hyper)",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "flate2"
 version = "0.2.14"
@@ -690,7 +701,7 @@ version = "0.1.0"
 dependencies = [
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)",
- "rustls 0.1.1 (git+https://github.com/ctz/rustls)",
+ "rustls 0.1.2 (git+https://github.com/ctz/rustls)",
 ]
 
 [[package]]
@@ -1345,7 +1356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "ring"
-version = "0.3.1"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1429,15 +1440,15 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.1.1"
+version = "0.1.2"
-source = "git+https://github.com/ctz/rustls#a9c5a79f49337e22ac05bb1ea114240bdbe0fdd2"
+source = "git+https://github.com/ctz/rustls#3d2db624997004b7b18ba4463d6081f37598b2f5"
 dependencies = [
  "base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1788,10 +1799,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "webpki"
-version = "0.2.2"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",

dapps/Cargo.toml

@@ -26,7 +26,7 @@ linked-hash-map = "0.3"
 ethcore-devtools = { path = "../devtools" }
 ethcore-rpc = { path = "../rpc" }
 ethcore-util = { path = "../util" }
-https-fetch = { path = "../util/https-fetch" }
+fetch = { path = "../util/fetch" }
 parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }
 # List of apps
 parity-dapps-status = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }

dapps/src/handlers/client/mod.rs

@@ -1,113 +0,0 @@
-// Copyright 2015, 2016 Ethcore (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Hyper Client Handlers
-
-pub mod fetch_file;
-
-use std::env;
-use std::sync::{mpsc, Arc};
-use std::sync::atomic::AtomicBool;
-use std::path::PathBuf;
-
-use hyper;
-use https_fetch as https;
-
-use random_filename;
-use self::fetch_file::{Fetch, Error as HttpFetchError};
-
-pub type FetchResult = Result<PathBuf, FetchError>;
-
-#[derive(Debug)]
-pub enum FetchError {
-	InvalidUrl,
-	Http(HttpFetchError),
-	Https(https::FetchError),
-	Other(String),
-}
-
-impl From<HttpFetchError> for FetchError {
-	fn from(e: HttpFetchError) -> Self {
-		FetchError::Http(e)
-	}
-}
-
-pub struct Client {
-	http_client: hyper::Client<Fetch>,
-	https_client: https::Client,
-}
-
-impl Client {
-	pub fn new() -> Self {
-		Client {
-			http_client: hyper::Client::new().expect("Unable to initialize http client."),
-			https_client: https::Client::new().expect("Unable to initialize https client."),
-		}
-	}
-
-	pub fn close(self) {
-		self.http_client.close();
-		self.https_client.close();
-	}
-
-	pub fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
-		let is_https = url.starts_with("https://");
-		let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
-		trace!(target: "dapps", "Fetching from: {:?}", url);
-		if is_https {
-			let url = try!(Self::convert_url(url));
-
-			let (tx, rx) = mpsc::channel();
-			let temp_path = Self::temp_path();
-			let res = self.https_client.fetch_to_file(url, temp_path.clone(), abort, move |result| {
-				let res = tx.send(
-					result.map(|_| temp_path).map_err(FetchError::Https)
-				);
-				if let Err(_) = res {
-					warn!("Fetch finished, but no one was listening");
-				}
-				on_done();
-			});
-
-			match res {
-				Ok(_) => Ok(rx),
-				Err(e) => Err(FetchError::Other(format!("{:?}", e))),
-			}
-		} else {
-			let (tx, rx) = mpsc::channel();
-			let res = self.http_client.request(url, Fetch::new(tx, abort, on_done));
-
-			match res {
-				Ok(_) => Ok(rx),
-				Err(e) => Err(FetchError::Other(format!("{:?}", e))),
-			}
-		}
-	}
-
-	fn convert_url(url: hyper::Url) -> Result<https::Url, FetchError> {
-		let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl)));
-		let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl));
-		https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl)
-	}
-
-	fn temp_path() -> PathBuf {
-		let mut dir = env::temp_dir();
-		dir.push(random_filename());
-		dir
-	}
-}
-
-

dapps/src/handlers/fetch.rs

@@ -22,13 +22,13 @@ use std::sync::{mpsc, Arc};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Instant, Duration};
 use util::Mutex;
+use fetch::{Client, Fetch, FetchResult};
 
 use hyper::{server, Decoder, Encoder, Next, Method, Control};
 use hyper::net::HttpStream;
 use hyper::status::StatusCode;
 
 use handlers::{ContentHandler, Redirection};
-use handlers::client::{Client, FetchResult};
 use apps::redirection_address;
 use page::LocalPageEndpoint;
 
@@ -159,7 +159,7 @@ impl<H: ContentValidator> ContentFetcherHandler<H> {
 		handler: H) -> (Self, Arc<FetchControl>) {
 
 		let fetch_control = Arc::new(FetchControl::default());
-		let client = Client::new();
+		let client = Client::default();
 		let handler = ContentFetcherHandler {
 			fetch_control: fetch_control.clone(),
 			control: Some(control),

dapps/src/handlers/mod.rs

@@ -21,7 +21,6 @@ mod echo;
 mod content;
 mod redirect;
 mod fetch;
-pub mod client;
 
 pub use self::auth::AuthRequiredHandler;
 pub use self::echo::EchoHandler;

dapps/src/lib.rs

@@ -58,10 +58,10 @@ extern crate jsonrpc_http_server;
 extern crate mime_guess;
 extern crate rustc_serialize;
 extern crate parity_dapps;
-extern crate https_fetch;
 extern crate ethcore_rpc;
 extern crate ethcore_util as util;
 extern crate linked_hash_map;
+extern crate fetch;
 #[cfg(test)]
 extern crate ethcore_devtools as devtools;
 

rpc/Cargo.toml

@@ -25,6 +25,7 @@ ethsync = { path = "../sync" }
 ethjson = { path = "../json" }
 ethcore-devtools = { path = "../devtools" }
 rlp = { path = "../util/rlp" }
+fetch = { path = "../util/fetch" }
 rustc-serialize = "0.3"
 transient-hashmap = "0.1"
 serde_macros = { version = "0.8.0", optional = true }

rpc/src/lib.rs

@@ -36,6 +36,7 @@ extern crate json_ipc_server as ipc;
 extern crate ethcore_ipc;
 extern crate time;
 extern crate rlp;
+extern crate fetch;
 
 #[macro_use]
 extern crate log;

rpc/src/v1/helpers/errors.rs

@@ -23,6 +23,7 @@ macro_rules! rpc_unimplemented {
 use std::fmt;
 use ethcore::error::Error as EthcoreError;
 use ethcore::account_provider::{Error as AccountError};
+use fetch::FetchError;
 use jsonrpc_core::{Error, ErrorCode, Value};
 
 mod codes {
@@ -41,6 +42,7 @@ mod codes {
 	pub const REQUEST_REJECTED_LIMIT: i64 = -32041;
 	pub const REQUEST_NOT_FOUND: i64 = -32042;
 	pub const COMPILATION_ERROR: i64 = -32050;
+	pub const FETCH_ERROR: i64 = -32060;
 }
 
 pub fn unimplemented() -> Error {
@@ -155,6 +157,14 @@ pub fn signer_disabled() -> Error {
 	}
 }
 
+pub fn from_fetch_error(error: FetchError) -> Error {
+	Error {
+		code: ErrorCode::ServerError(codes::FETCH_ERROR),
+		message: "Error while fetching content.".into(),
+		data: Some(Value::String(format!("{:?}", error))),
+	}
+}
+
 pub fn from_signing_error(error: AccountError) -> Error {
 	Error {
 		code: ErrorCode::ServerError(codes::ACCOUNT_LOCKED),

rpc/src/v1/impls/ethcore.rs

@@ -15,30 +15,33 @@
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Ethcore-specific rpc implementation.
-use std::sync::{Arc, Weak};
+use std::{fs, io};
+use std::sync::{mpsc, Arc, Weak};
 use std::str::FromStr;
 use std::collections::{BTreeMap};
-use util::{RotatingLogger, Address};
+use util::{RotatingLogger, Address, Mutex, sha3};
 use util::misc::version_data;
 
 use crypto::ecies;
+use fetch::{Client as FetchClient, Fetch};
 use ethkey::{Brain, Generator};
 use ethstore::random_phrase;
 use ethsync::{SyncProvider, ManageNetwork};
 use ethcore::miner::MinerService;
 use ethcore::client::{MiningBlockChainClient};
 
-use jsonrpc_core::*;
+use jsonrpc_core::{from_params, to_value, Value, Error, Params, Ready};
 use v1::traits::Ethcore;
-use v1::types::{Bytes, U256, H160, H512, Peers, Transaction};
+use v1::types::{Bytes, U256, H160, H256, H512, Peers, Transaction};
 use v1::helpers::{errors, SigningQueue, SignerService, NetworkSettings};
 use v1::helpers::params::expect_no_params;
 
 /// Ethcore implementation.
-pub struct EthcoreClient<C, M, S: ?Sized> where
+pub struct EthcoreClient<C, M, S: ?Sized, F=FetchClient> where
 	C: MiningBlockChainClient,
 	M: MinerService,
-	S: SyncProvider {
+	S: SyncProvider,
+	F: Fetch {
 
 	client: Weak<C>,
 	miner: Weak<M>,
@@ -47,10 +50,14 @@ pub struct EthcoreClient<C, M, S: ?Sized> where
 	logger: Arc<RotatingLogger>,
 	settings: Arc<NetworkSettings>,
 	signer: Option<Arc<SignerService>>,
+	fetch: Mutex<F>
 }
 
-impl<C, M, S: ?Sized> EthcoreClient<C, M, S> where C: MiningBlockChainClient, M: MinerService, S: SyncProvider {
+impl<C, M, S: ?Sized> EthcoreClient<C, M, S> where
-	/// Creates new `EthcoreClient`.
+	C: MiningBlockChainClient,
+	M: MinerService,
+	S: SyncProvider, {
+	/// Creates new `EthcoreClient` with default `Fetch`.
 	pub fn new(
 		client: &Arc<C>,
 		miner: &Arc<M>,
@@ -59,6 +66,26 @@ impl<C, M, S: ?Sized> EthcoreClient<C, M, S> where C: MiningBlockChainClient, M:
 		logger: Arc<RotatingLogger>,
 		settings: Arc<NetworkSettings>,
 		signer: Option<Arc<SignerService>>
+	) -> Self {
+		Self::with_fetch(client, miner, sync, net, logger, settings, signer)
+	}
+}
+
+impl<C, M, S: ?Sized, F> EthcoreClient<C, M, S, F> where
+	C: MiningBlockChainClient,
+	M: MinerService,
+	S: SyncProvider,
+	F: Fetch, {
+
+	/// Creates new `EthcoreClient` with customizable `Fetch`.
+	pub fn with_fetch(
+		client: &Arc<C>,
+		miner: &Arc<M>,
+		sync: &Arc<S>,
+		net: &Arc<ManageNetwork>,
+		logger: Arc<RotatingLogger>,
+		settings: Arc<NetworkSettings>,
+		signer: Option<Arc<SignerService>>
 		) -> Self {
 		EthcoreClient {
 			client: Arc::downgrade(client),
@@ -68,6 +95,7 @@ impl<C, M, S: ?Sized> EthcoreClient<C, M, S> where C: MiningBlockChainClient, M:
 			logger: logger,
 			settings: settings,
 			signer: signer,
+			fetch: Mutex::new(F::default()),
 		}
 	}
 
@@ -78,7 +106,11 @@ impl<C, M, S: ?Sized> EthcoreClient<C, M, S> where C: MiningBlockChainClient, M:
 	}
 }
 
-impl<C, M, S: ?Sized> Ethcore for EthcoreClient<C, M, S> where M: MinerService + 'static, C: MiningBlockChainClient + 'static, S: SyncProvider + 'static {
+impl<C, M, S: ?Sized, F> Ethcore for EthcoreClient<C, M, S, F> where
+	M: MinerService + 'static,
+	C: MiningBlockChainClient + 'static,
+	S: SyncProvider + 'static,
+	F: Fetch + 'static {
 
 	fn transactions_limit(&self, params: Params) -> Result<Value, Error> {
 		try!(self.active());
@@ -233,4 +265,42 @@ impl<C, M, S: ?Sized> Ethcore for EthcoreClient<C, M, S> where M: MinerService +
 
 		Ok(to_value(&take_weak!(self.miner).all_transactions().into_iter().map(Into::into).collect::<Vec<Transaction>>()))
 	}
+
+	fn hash_content(&self, params: Params, ready: Ready) {
+		let res = self.active().and_then(|_| from_params::<(String,)>(params));
+
+		let hash_content = |result| {
+			let path = try!(result);
+			let mut file = io::BufReader::new(try!(fs::File::open(&path)));
+			// Try to hash
+			let result = sha3(&mut file);
+			// Remove file (always)
+			try!(fs::remove_file(&path));
+			// Return the result
+			Ok(try!(result))
+		};
+
+		match res {
+			Err(e) => ready.ready(Err(e)),
+			Ok((url, )) => {
+				let (tx, rx) = mpsc::channel();
+				let res = self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| {
+					let result = hash_content(result)
+							.map_err(errors::from_fetch_error)
+							.map(|hash| to_value(H256::from(hash)));
+
+					// Receive ready and invoke with result.
+					let ready: Ready = rx.try_recv().expect("When on_done is invoked ready object is always sent.");
+					ready.ready(result);
+				}));
+
+				// Either invoke ready right away or transfer it to the closure.
+				if let Err(e) = res {
+					ready.ready(Err(errors::from_fetch_error(e)));
+				} else {
+					tx.send(ready).expect("Rx end is sent to on_done closure.");
+				}
+			}
+		}
+	}
 }

rpc/src/v1/tests/helpers/fetch.rs

@@ -0,0 +1,44 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test implementation of fetch client.
+
+use std::io::Write;
+use std::{env, fs, thread};
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+use fetch::{Fetch, FetchError, FetchResult};
+
+/// Test implementation of fetcher. Will always return the same file.
+#[derive(Default)]
+pub struct TestFetch;
+
+impl Fetch for TestFetch {
+	fn request_async(&mut self, _url: &str, _abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError> {
+		thread::spawn(move || {
+			let mut path = env::temp_dir();
+			path.push(Self::random_filename());
+
+			let mut file = fs::File::create(&path).unwrap();
+			file.write_all(b"Some content").unwrap();
+
+			on_done(Ok(path));
+		});
+		Ok(())
+	}
+}
+
+

rpc/src/v1/tests/helpers/mod.rs

@@ -18,6 +18,8 @@
 
 mod sync_provider;
 mod miner_service;
+mod fetch;
 
 pub use self::sync_provider::{Config, TestSyncProvider};
 pub use self::miner_service::TestMinerService;
+pub use self::fetch::TestFetch;

rpc/src/v1/tests/mocked/ethcore.rs

@@ -23,7 +23,7 @@ use ethcore::client::{TestBlockChainClient};
 use jsonrpc_core::IoHandler;
 use v1::{Ethcore, EthcoreClient};
 use v1::helpers::{SignerService, NetworkSettings};
-use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService};
+use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService, TestFetch};
 use super::manage_network::TestManageNetwork;
 
 fn miner_service() -> Arc<TestMinerService> {
@@ -60,12 +60,15 @@ fn network_service() -> Arc<ManageNetwork> {
 	Arc::new(TestManageNetwork)
 }
 
+type TestEthcoreClient = EthcoreClient<TestBlockChainClient, TestMinerService, TestSyncProvider, TestFetch>;
+
 fn ethcore_client(
 	client: &Arc<TestBlockChainClient>,
 	miner: &Arc<TestMinerService>,
 	sync: &Arc<TestSyncProvider>,
-	net: &Arc<ManageNetwork>) -> EthcoreClient<TestBlockChainClient, TestMinerService, TestSyncProvider> {
+	net: &Arc<ManageNetwork>)
-	EthcoreClient::new(client, miner, sync, net, logger(), settings(), None)
+	-> TestEthcoreClient {
+	EthcoreClient::with_fetch(client, miner, sync, net, logger(), settings(), None)
 }
 
 #[test]
@@ -140,9 +143,9 @@ fn rpc_ethcore_dev_logs() {
 	let logger = logger();
 	logger.append("a".to_owned());
 	logger.append("b".to_owned());
-	let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger.clone(), settings(), None).to_delegate();
+	let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger.clone(), settings(), None);
 	let io = IoHandler::new();
-	io.add_delegate(ethcore);
+	io.add_delegate(ethcore.to_delegate());
 
 	let request = r#"{"jsonrpc": "2.0", "method": "ethcore_devLogs", "params":[], "id": 1}"#;
 	let response = r#"{"jsonrpc":"2.0","result":["b","a"],"id":1}"#;
@@ -263,8 +266,8 @@ fn rpc_ethcore_unsigned_transactions_count() {
 	let net = network_service();
 	let io = IoHandler::new();
 	let signer = Arc::new(SignerService::new_test());
-	let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger(), settings(), Some(signer)).to_delegate();
+	let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger(), settings(), Some(signer));
-	io.add_delegate(ethcore);
+	io.add_delegate(ethcore.to_delegate());
 
 	let request = r#"{"jsonrpc": "2.0", "method": "ethcore_unsignedTransactionsCount", "params":[], "id": 1}"#;
 	let response = r#"{"jsonrpc":"2.0","result":0,"id":1}"#;
@@ -287,6 +290,21 @@ fn rpc_ethcore_unsigned_transactions_count_when_signer_disabled() {
 	assert_eq!(io.handle_request_sync(request), Some(response.to_owned()));
 }
 
+#[test]
+fn rpc_ethcore_hash_content() {
+	let miner = miner_service();
+	let client = client_service();
+	let sync = sync_provider();
+	let net = network_service();
+	let io = IoHandler::new();
+	io.add_delegate(ethcore_client(&client, &miner, &sync, &net).to_delegate());
+
+	let request = r#"{"jsonrpc": "2.0", "method": "ethcore_hashContent", "params":["https://ethcore.io/assets/images/ethcore-black-horizontal.png"], "id": 1}"#;
+	let response = r#"{"jsonrpc":"2.0","result":"0x2be00befcf008bc0e7d9cdefc194db9c75352e8632f48498b5a6bfce9f02c88e","id":1}"#;
+
+	assert_eq!(io.handle_request_sync(request), Some(response.to_owned()));
+}
+
 #[test]
 fn rpc_ethcore_pending_transactions() {
 	let miner = miner_service();

rpc/src/v1/traits/ethcore.rs

@@ -83,6 +83,9 @@ pub trait Ethcore: Sized + Send + Sync + 'static {
 	/// Returns all pending (current) transactions from transaction queue.
 	fn pending_transactions(&self, _: Params) -> Result<Value, Error>;
 
+	/// Hash a file content under given URL.
+	fn hash_content(&self, _: Params, _: Ready);
+
 	/// Should be used to convert object to io delegate.
 	fn to_delegate(self) -> IoDelegate<Self> {
 		let mut delegate = IoDelegate::new(Arc::new(self));
@@ -107,6 +110,8 @@ pub trait Ethcore: Sized + Send + Sync + 'static {
 		delegate.add_method("ethcore_registryAddress", Ethcore::registry_address);
 		delegate.add_method("ethcore_encryptMessage", Ethcore::encrypt_message);
 		delegate.add_method("ethcore_pendingTransactions", Ethcore::pending_transactions);
+		delegate.add_async_method("ethcore_hashContent", Ethcore::hash_content);
+
 		delegate
 	}
 }

util/fetch/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+description = "HTTP/HTTPS fetching library"
+homepage = "http://ethcore.io"
+license = "GPL-3.0"
+name = "fetch"
+version = "0.1.0"
+authors = ["Ethcore <admin@ethcore.io>"]
+
+[dependencies]
+log = "0.3"
+rand = "0.3"
+hyper = { default-features = false, git = "https://github.com/ethcore/hyper" }
+https-fetch = { path = "../https-fetch" }
+clippy = { version = "0.0.90", optional = true}
+
+[features]
+default = []
+dev = ["clippy"]

util/fetch/src/client.rs

@@ -0,0 +1,146 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Fetching
+
+use std::{env, io};
+use std::sync::{mpsc, Arc};
+use std::sync::atomic::AtomicBool;
+use std::path::PathBuf;
+
+use hyper;
+use https_fetch as https;
+
+use fetch_file::{FetchHandler, Error as HttpFetchError};
+
+pub type FetchResult = Result<PathBuf, FetchError>;
+
+#[derive(Debug)]
+pub enum FetchError {
+	InvalidUrl,
+	Http(HttpFetchError),
+	Https(https::FetchError),
+	Io(io::Error),
+	Other(String),
+}
+
+impl From<HttpFetchError> for FetchError {
+	fn from(e: HttpFetchError) -> Self {
+		FetchError::Http(e)
+	}
+}
+
+impl From<io::Error> for FetchError {
+	fn from(e: io::Error) -> Self {
+		FetchError::Io(e)
+	}
+}
+
+pub trait Fetch: Default + Send {
+	/// Fetch URL and get the result in callback.
+	fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError>;
+
+	/// Fetch URL and get a result Receiver. You will be notified when receiver is ready by `on_done` callback.
+	fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
+		let (tx, rx) = mpsc::channel();
+		try!(self.request_async(url, abort, Box::new(move |result| {
+			let res = tx.send(result);
+			if let Err(_) = res {
+				warn!("Fetch finished, but no one was listening");
+			}
+			on_done();
+		})));
+		Ok(rx)
+	}
+
+	/// Closes this client
+	fn close(self) {}
+
+	/// Returns a random filename
+	fn random_filename() -> String {
+		use ::rand::Rng;
+		let mut rng = ::rand::OsRng::new().unwrap();
+		rng.gen_ascii_chars().take(12).collect()
+	}
+}
+
+pub struct Client {
+	http_client: hyper::Client<FetchHandler>,
+	https_client: https::Client,
+	limit: Option<usize>,
+}
+
+impl Default for Client {
+	fn default() -> Self {
+		// Max 15MB will be downloaded.
+		Client::with_limit(Some(15*1024*1024))
+	}
+}
+
+impl Client {
+	fn with_limit(limit: Option<usize>) -> Self {
+		Client {
+			http_client: hyper::Client::new().expect("Unable to initialize http client."),
+			https_client: https::Client::with_limit(limit).expect("Unable to initialize https client."),
+			limit: limit,
+		}
+	}
+
+	fn convert_url(url: hyper::Url) -> Result<https::Url, FetchError> {
+		let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl)));
+		let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl));
+		https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl)
+	}
+
+	fn temp_path() -> PathBuf {
+		let mut dir = env::temp_dir();
+		dir.push(Self::random_filename());
+		dir
+	}
+}
+
+impl Fetch for Client {
+	fn close(self) {
+		self.http_client.close();
+		self.https_client.close();
+	}
+
+	fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError> {
+		let is_https = url.starts_with("https://");
+		let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
+		let temp_path = Self::temp_path();
+
+		trace!(target: "fetch", "Fetching from: {:?}", url);
+
+		if is_https {
+			let url = try!(Self::convert_url(url));
+			try!(self.https_client.fetch_to_file(
+				url,
+				temp_path.clone(),
+				abort,
+				move |result| on_done(result.map(|_| temp_path).map_err(FetchError::Https)),
+			).map_err(|e| FetchError::Other(format!("{:?}", e))));
+		} else {
+			try!(self.http_client.request(
+				url,
+				FetchHandler::new(temp_path, abort, Box::new(move |result| on_done(result)), self.limit.map(|v| v as u64).clone()),
+			).map_err(|e| FetchError::Other(format!("{:?}", e))));
+		}
+
+		Ok(())
+	}
+}
+

util/fetch/src/fetch_file.rs

@@ -16,12 +16,11 @@
 
 //! Hyper Client Handler to Fetch File
 
-use std::{env, io, fs, fmt};
+use std::{io, fs, fmt};
 use std::path::PathBuf;
-use std::sync::{mpsc, Arc};
+use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
-use random_filename;
 
 use hyper::status::StatusCode;
 use hyper::client::{Request, Response, DefaultTransport as HttpStream};
@@ -34,30 +33,31 @@ use super::FetchError;
 pub enum Error {
 	Aborted,
 	NotStarted,
+	SizeLimit,
 	UnexpectedStatus(StatusCode),
 	IoError(io::Error),
 	HyperError(hyper::Error),
 }
 
 pub type FetchResult = Result<PathBuf, FetchError>;
-pub type OnDone = Box<Fn() + Send>;
+pub type OnDone = Box<Fn(FetchResult) + Send>;
 
-pub struct Fetch {
+pub struct FetchHandler {
 	path: PathBuf,
 	abort: Arc<AtomicBool>,
 	file: Option<fs::File>,
 	result: Option<FetchResult>,
-	sender: mpsc::Sender<FetchResult>,
 	on_done: Option<OnDone>,
+	size_limit: Option<u64>,
 }
 
-impl fmt::Debug for Fetch {
+impl fmt::Debug for FetchHandler {
 	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
 		write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result)
 	}
 }
 
-impl Drop for Fetch {
+impl Drop for FetchHandler {
     fn drop(&mut self) {
 		let res = self.result.take().unwrap_or(Err(Error::NotStarted.into()));
 		// Remove file if there was an error
@@ -69,40 +69,35 @@ impl Drop for Fetch {
 			}
 		}
 		// send result
-		let _ = self.sender.send(res);
 		if let Some(f) = self.on_done.take() {
-			f();
+			f(res);
 		}
     }
 }
 
-impl Fetch {
+impl FetchHandler {
-	pub fn new(sender: mpsc::Sender<FetchResult>, abort: Arc<AtomicBool>, on_done: OnDone) -> Self {
+	pub fn new(path: PathBuf, abort: Arc<AtomicBool>, on_done: OnDone, size_limit: Option<u64>) -> Self {
-		let mut dir = env::temp_dir();
+		FetchHandler {
-		dir.push(random_filename());
+			path: path,
-
-		Fetch {
-			path: dir,
 			abort: abort,
 			file: None,
 			result: None,
-			sender: sender,
 			on_done: Some(on_done),
+			size_limit: size_limit,
 		}
 	}
-}
 
-impl Fetch {
 	fn is_aborted(&self) -> bool {
 		self.abort.load(Ordering::SeqCst)
 	}
+
 	fn mark_aborted(&mut self) -> Next {
 		self.result = Some(Err(Error::Aborted.into()));
 		Next::end()
 	}
 }
 
-impl hyper::client::Handler<HttpStream> for Fetch {
+impl hyper::client::Handler<HttpStream> for FetchHandler {
     fn on_request(&mut self, req: &mut Request) -> Next {
 		if self.is_aborted() {
 			return self.mark_aborted();
@@ -147,7 +142,19 @@ impl hyper::client::Handler<HttpStream> for Fetch {
 		}
         match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) {
             Ok(0) => Next::end(),
-            Ok(_) => read(),
+            Ok(bytes_read) => match self.size_limit {
+				None => read(),
+				// Check limit
+				Some(limit) if limit > bytes_read => {
+					self.size_limit = Some(limit - bytes_read);
+					read()
+				},
+				// Size limit reached
+				_ => {
+					self.result = Some(Err(Error::SizeLimit.into()));
+					Next::end()
+				},
+			},
             Err(e) => match e.kind() {
                 io::ErrorKind::WouldBlock => Next::read(),
                 _ => {

util/fetch/src/lib.rs

@@ -0,0 +1,29 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A service to fetch any HTTP / HTTPS content.
+
+#[macro_use]
+extern crate log;
+extern crate hyper;
+extern crate https_fetch;
+extern crate rand;
+
+
+pub mod client;
+pub mod fetch_file;
+
+pub use self::client::{Client, Fetch, FetchError, FetchResult};

util/https-fetch/src/client.rs

@@ -78,6 +78,10 @@ impl Drop for Client {
 
 impl Client {
 	pub fn new() -> Result<Self, FetchError> {
+		Self::with_limit(None)
+	}
+
+	pub fn with_limit(size_limit: Option<usize>) -> Result<Self, FetchError> {
 		let mut event_loop = try!(mio::EventLoop::new());
 		let channel = event_loop.channel();
 
@@ -85,6 +89,7 @@ impl Client {
 			let mut client = ClientLoop {
 				next_token: 0,
 				sessions: HashMap::new(),
+				size_limit: size_limit,
 			};
 			event_loop.run(&mut client).unwrap();
 		});
@@ -128,6 +133,7 @@ impl Client {
 pub struct ClientLoop {
 	next_token: usize,
 	sessions: HashMap<usize, TlsClient>,
+	size_limit: Option<usize>,
 }
 
 impl mio::Handler for ClientLoop {
@@ -154,7 +160,7 @@ impl mio::Handler for ClientLoop {
 				let token = self.next_token;
 				self.next_token += 1;
 
-				if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback) {
+				if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback, self.size_limit.clone()) {
 					let httpreq = format!(
 						"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n",
 						url.path(),

util/https-fetch/src/http.rs

@@ -35,18 +35,20 @@ pub struct HttpProcessor {
 	status: Option<String>,
 	headers: Vec<String>,
 	body_writer: io::BufWriter<Box<io::Write>>,
+	size_limit: Option<usize>,
 }
 
 const BREAK_LEN: usize = 2;
 
 impl HttpProcessor {
-	pub fn new(body_writer: Box<io::Write>) -> Self {
+	pub fn new(body_writer: Box<io::Write>, size_limit: Option<usize>) -> Self {
 		HttpProcessor {
 			state: State::WaitingForStatus,
 			buffer: Cursor::new(Vec::new()),
 			status: None,
 			headers: Vec::new(),
-			body_writer: io::BufWriter::new(body_writer)
+			body_writer: io::BufWriter::new(body_writer),
+			size_limit: size_limit,
 		}
 	}
 
@@ -140,6 +142,15 @@ impl HttpProcessor {
 				},
 				State::WritingBody => {
 					let len = self.buffer.get_ref().len();
+					match self.size_limit {
+						None => {},
+						Some(limit) if limit > len => {},
+						_ => {
+							warn!("Finishing file fetching because limit was reached.");
+							self.set_state(State::Finished);
+							continue;
+						}
+					}
 					try!(self.body_writer.write_all(self.buffer.get_ref()));
 					self.buffer_consume(len);
 					return Ok(());
@@ -167,6 +178,17 @@ impl HttpProcessor {
 				},
 				// Buffers the data until we have a full chunk
 				State::WritingChunk(left) if self.buffer.get_ref().len() >= left => {
+					match self.size_limit {
+						None => {},
+						Some(limit) if limit > left => {
+							self.size_limit = Some(limit - left);
+						},
+						_ => {
+							warn!("Finishing file fetching because limit was reached.");
+							self.set_state(State::Finished);
+							continue;
+						}
+					}
 					try!(self.body_writer.write_all(&self.buffer.get_ref()[0..left]));
 					self.buffer_consume(left + BREAK_LEN);
 
@@ -230,7 +252,7 @@ mod tests {
 	#[test]
 	fn should_be_able_to_process_status_line() {
 		// given
-		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
 
 		// when
 		let out =
@@ -249,7 +271,7 @@ mod tests {
 	#[test]
 	fn should_be_able_to_process_headers() {
 		// given
-		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())));
+		let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
 
 		// when
 		let out =
@@ -274,7 +296,7 @@ mod tests {
 	fn should_be_able_to_consume_body() {
 		// given
 		let (writer, data) = Writer::new();
-		let mut http = HttpProcessor::new(Box::new(writer));
+		let mut http = HttpProcessor::new(Box::new(writer), None);
 
 		// when
 		let out =
@@ -301,7 +323,7 @@ mod tests {
 	fn should_correctly_handle_chunked_content() {
 		// given
 		let (writer, data) = Writer::new();
-		let mut http = HttpProcessor::new(Box::new(writer));
+		let mut http = HttpProcessor::new(Box::new(writer), None);
 
 		// when
 		let out =
@@ -331,4 +353,40 @@ mod tests {
 		assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]);
 		assert_eq!(http.state(), State::Finished);
 	}
+
+	#[test]
+	fn should_stop_fetching_when_limit_is_reached() {
+		// given
+		let (writer, data) = Writer::new();
+		let mut http = HttpProcessor::new(Box::new(writer), Some(5));
+
+		// when
+		let out =
+			"\
+				HTTP/1.1 200 OK\r\n\
+				Host: 127.0.0.1:8080\r\n\
+				Transfer-Encoding: chunked\r\n\
+				Connection: close\r\n\
+				\r\n\
+				4\r\n\
+				Pari\r\n\
+				3\r\n\
+				ty \r\n\
+				D\r\n\
+				in\r\n\
+				\r\n\
+				chunks.\r\n\
+				0\r\n\
+				\r\n\
+			";
+		http.write_all(out.as_bytes()).unwrap();
+		http.flush().unwrap();
+
+		// then
+		assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
+		assert_eq!(http.headers().len(), 3);
+		assert_eq!(data.borrow().get_ref()[..], b"Pari"[..]);
+		assert_eq!(http.state(), State::Finished);
+	}
+
 }

util/https-fetch/src/tlsclient.rs

@@ -87,6 +87,7 @@ impl TlsClient {
 		writer: Box<io::Write + Send>,
 		abort: Arc<AtomicBool>,
 		mut callback: Box<FnMut(FetchResult) + Send>,
+		size_limit: Option<usize>,
 		) -> Result<Self, FetchError> {
 			let res = TlsClient::make_config().and_then(|cfg| {
 				TcpStream::connect(url.address()).map(|sock| {
@@ -98,7 +99,7 @@ impl TlsClient {
 				Ok((cfg, sock)) => Ok(TlsClient {
 					abort: abort,
 					token: token,
-					writer: HttpProcessor::new(writer),
+					writer: HttpProcessor::new(writer, size_limit),
 					socket: sock,
 					closing: false,
 					error: None,