From d7bbc5cc3f7cc1d6c6df34c34d44043345c2a3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 27 Sep 2016 16:27:06 +0200 Subject: [PATCH] Hash Content RPC method (#2355) * Moving file fetching to separate crate. * ethcore_hashContent * Tests running on mocked fetch. * Limiting size of downloadable assets --- Cargo.lock | 29 ++-- dapps/Cargo.toml | 2 +- dapps/src/handlers/client/mod.rs | 113 -------------- dapps/src/handlers/fetch.rs | 4 +- dapps/src/handlers/mod.rs | 1 - dapps/src/lib.rs | 2 +- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/helpers/errors.rs | 10 ++ rpc/src/v1/impls/ethcore.rs | 88 +++++++++-- rpc/src/v1/tests/helpers/fetch.rs | 44 ++++++ rpc/src/v1/tests/helpers/mod.rs | 2 + rpc/src/v1/tests/mocked/ethcore.rs | 32 +++- rpc/src/v1/traits/ethcore.rs | 5 + util/fetch/Cargo.toml | 18 +++ util/fetch/src/client.rs | 146 ++++++++++++++++++ .../client => util/fetch/src}/fetch_file.rs | 51 +++--- util/fetch/src/lib.rs | 29 ++++ util/https-fetch/src/client.rs | 8 +- util/https-fetch/src/http.rs | 70 ++++++++- util/https-fetch/src/tlsclient.rs | 3 +- 21 files changed, 486 insertions(+), 173 deletions(-) delete mode 100644 dapps/src/handlers/client/mod.rs create mode 100644 rpc/src/v1/tests/helpers/fetch.rs create mode 100644 util/fetch/Cargo.toml create mode 100644 util/fetch/src/client.rs rename {dapps/src/handlers/client => util/fetch/src}/fetch_file.rs (82%) create mode 100644 util/fetch/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 52bdee494..2ae27d202 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,7 +317,7 @@ dependencies = [ "ethcore-devtools 1.4.0", "ethcore-rpc 1.4.0", "ethcore-util 1.4.0", - "https-fetch 0.1.0", + "fetch 0.1.0", "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)", @@ -466,6 +466,7 @@ dependencies = [ "ethkey 0.2.0", "ethstore 0.1.0", "ethsync 1.4.0", + "fetch 0.1.0", "json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)", "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)", @@ -639,6 +640,16 @@ dependencies = [ "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fetch" +version = "0.1.0" +dependencies = [ + "https-fetch 0.1.0", + "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flate2" version = "0.2.14" @@ -690,7 +701,7 @@ version = "0.1.0" dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)", - "rustls 0.1.1 (git+https://github.com/ctz/rustls)", + "rustls 0.1.2 (git+https://github.com/ctz/rustls)", ] [[package]] @@ -1345,7 +1356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ring" -version = "0.3.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1429,15 +1440,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.1.1" -source = "git+https://github.com/ctz/rustls#a9c5a79f49337e22ac05bb1ea114240bdbe0fdd2" +version = "0.1.2" +source = "git+https://github.com/ctz/rustls#3d2db624997004b7b18ba4463d6081f37598b2f5" dependencies = [ "base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1788,10 +1799,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "webpki" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index b1883e748..1019e5460 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -26,7 +26,7 @@ linked-hash-map = "0.3" ethcore-devtools = { path = "../devtools" } ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } -https-fetch = { path = "../util/https-fetch" } +fetch = { path = "../util/fetch" } parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } # List of apps parity-dapps-status = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } diff --git a/dapps/src/handlers/client/mod.rs b/dapps/src/handlers/client/mod.rs deleted file mode 100644 index 3d8551e8a..000000000 --- a/dapps/src/handlers/client/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Hyper Client Handlers - -pub mod fetch_file; - -use std::env; -use std::sync::{mpsc, Arc}; -use std::sync::atomic::AtomicBool; -use std::path::PathBuf; - -use hyper; -use https_fetch as https; - -use random_filename; -use self::fetch_file::{Fetch, Error as HttpFetchError}; - -pub type FetchResult = Result; - -#[derive(Debug)] -pub enum FetchError { - InvalidUrl, - Http(HttpFetchError), - Https(https::FetchError), - Other(String), -} - -impl From for FetchError { - fn from(e: HttpFetchError) -> Self { - FetchError::Http(e) - } -} - -pub struct Client { - http_client: hyper::Client, - https_client: https::Client, -} - -impl Client { - pub fn new() -> Self { - Client { - http_client: hyper::Client::new().expect("Unable to initialize http client."), - https_client: https::Client::new().expect("Unable to initialize https client."), - } - } - - pub fn close(self) { - self.http_client.close(); - self.https_client.close(); - } - - pub fn request(&mut self, url: &str, abort: Arc, on_done: Box) -> Result, FetchError> { - let is_https = url.starts_with("https://"); - let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl)); - trace!(target: "dapps", "Fetching from: {:?}", url); - if is_https { - let url = try!(Self::convert_url(url)); - - let (tx, rx) = mpsc::channel(); - let temp_path = Self::temp_path(); - let res = self.https_client.fetch_to_file(url, temp_path.clone(), abort, move |result| { - let res = tx.send( - result.map(|_| temp_path).map_err(FetchError::Https) - ); - if let Err(_) = res { - warn!("Fetch finished, but no one was listening"); - } - on_done(); - }); - - match res { - Ok(_) => Ok(rx), - Err(e) => Err(FetchError::Other(format!("{:?}", e))), - } - } else { - let (tx, rx) = mpsc::channel(); - let res = self.http_client.request(url, Fetch::new(tx, abort, on_done)); - - match res { - Ok(_) => Ok(rx), - Err(e) => Err(FetchError::Other(format!("{:?}", e))), - } - } - } - - fn convert_url(url: hyper::Url) -> Result { - let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl))); - let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl)); - https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl) - } - - fn temp_path() -> PathBuf { - let mut dir = env::temp_dir(); - dir.push(random_filename()); - dir - } -} - - diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index c463d3710..639fc7497 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -22,13 +22,13 @@ use std::sync::{mpsc, Arc}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Instant, Duration}; use util::Mutex; +use fetch::{Client, Fetch, FetchResult}; use hyper::{server, Decoder, Encoder, Next, Method, Control}; use hyper::net::HttpStream; use hyper::status::StatusCode; use handlers::{ContentHandler, Redirection}; -use handlers::client::{Client, FetchResult}; use apps::redirection_address; use page::LocalPageEndpoint; @@ -159,7 +159,7 @@ impl ContentFetcherHandler { handler: H) -> (Self, Arc) { let fetch_control = Arc::new(FetchControl::default()); - let client = Client::new(); + let client = Client::default(); let handler = ContentFetcherHandler { fetch_control: fetch_control.clone(), control: Some(control), diff --git a/dapps/src/handlers/mod.rs b/dapps/src/handlers/mod.rs index 62b13eaa8..54644fe8d 100644 --- a/dapps/src/handlers/mod.rs +++ b/dapps/src/handlers/mod.rs @@ -21,7 +21,6 @@ mod echo; mod content; mod redirect; mod fetch; -pub mod client; pub use self::auth::AuthRequiredHandler; pub use self::echo::EchoHandler; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index edc0bebe5..cac42f893 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -58,10 +58,10 @@ extern crate jsonrpc_http_server; extern crate mime_guess; extern crate rustc_serialize; extern crate parity_dapps; -extern crate https_fetch; extern crate ethcore_rpc; extern crate ethcore_util as util; extern crate linked_hash_map; +extern crate fetch; #[cfg(test)] extern crate ethcore_devtools as devtools; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c3f9cddbd..34b68fb81 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -25,6 +25,7 @@ ethsync = { path = "../sync" } ethjson = { path = "../json" } ethcore-devtools = { path = "../devtools" } rlp = { path = "../util/rlp" } +fetch = { path = "../util/fetch" } rustc-serialize = "0.3" transient-hashmap = "0.1" serde_macros = { version = "0.8.0", optional = true } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 7f2f11400..01ba44941 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -36,6 +36,7 @@ extern crate json_ipc_server as ipc; extern crate ethcore_ipc; extern crate time; extern crate rlp; +extern crate fetch; #[macro_use] extern crate log; diff --git a/rpc/src/v1/helpers/errors.rs b/rpc/src/v1/helpers/errors.rs index 18e369208..885ec08f0 100644 --- a/rpc/src/v1/helpers/errors.rs +++ b/rpc/src/v1/helpers/errors.rs @@ -23,6 +23,7 @@ macro_rules! rpc_unimplemented { use std::fmt; use ethcore::error::Error as EthcoreError; use ethcore::account_provider::{Error as AccountError}; +use fetch::FetchError; use jsonrpc_core::{Error, ErrorCode, Value}; mod codes { @@ -41,6 +42,7 @@ mod codes { pub const REQUEST_REJECTED_LIMIT: i64 = -32041; pub const REQUEST_NOT_FOUND: i64 = -32042; pub const COMPILATION_ERROR: i64 = -32050; + pub const FETCH_ERROR: i64 = -32060; } pub fn unimplemented() -> Error { @@ -155,6 +157,14 @@ pub fn signer_disabled() -> Error { } } +pub fn from_fetch_error(error: FetchError) -> Error { + Error { + code: ErrorCode::ServerError(codes::FETCH_ERROR), + message: "Error while fetching content.".into(), + data: Some(Value::String(format!("{:?}", error))), + } +} + pub fn from_signing_error(error: AccountError) -> Error { Error { code: ErrorCode::ServerError(codes::ACCOUNT_LOCKED), diff --git a/rpc/src/v1/impls/ethcore.rs b/rpc/src/v1/impls/ethcore.rs index 220ead3dd..684ce61a4 100644 --- a/rpc/src/v1/impls/ethcore.rs +++ b/rpc/src/v1/impls/ethcore.rs @@ -15,30 +15,33 @@ // along with Parity. If not, see . //! Ethcore-specific rpc implementation. -use std::sync::{Arc, Weak}; +use std::{fs, io}; +use std::sync::{mpsc, Arc, Weak}; use std::str::FromStr; use std::collections::{BTreeMap}; -use util::{RotatingLogger, Address}; +use util::{RotatingLogger, Address, Mutex, sha3}; use util::misc::version_data; use crypto::ecies; +use fetch::{Client as FetchClient, Fetch}; use ethkey::{Brain, Generator}; use ethstore::random_phrase; use ethsync::{SyncProvider, ManageNetwork}; use ethcore::miner::MinerService; use ethcore::client::{MiningBlockChainClient}; -use jsonrpc_core::*; +use jsonrpc_core::{from_params, to_value, Value, Error, Params, Ready}; use v1::traits::Ethcore; -use v1::types::{Bytes, U256, H160, H512, Peers, Transaction}; +use v1::types::{Bytes, U256, H160, H256, H512, Peers, Transaction}; use v1::helpers::{errors, SigningQueue, SignerService, NetworkSettings}; use v1::helpers::params::expect_no_params; /// Ethcore implementation. -pub struct EthcoreClient where +pub struct EthcoreClient where C: MiningBlockChainClient, M: MinerService, - S: SyncProvider { + S: SyncProvider, + F: Fetch { client: Weak, miner: Weak, @@ -47,10 +50,14 @@ pub struct EthcoreClient where logger: Arc, settings: Arc, signer: Option>, + fetch: Mutex } -impl EthcoreClient where C: MiningBlockChainClient, M: MinerService, S: SyncProvider { - /// Creates new `EthcoreClient`. +impl EthcoreClient where + C: MiningBlockChainClient, + M: MinerService, + S: SyncProvider, { + /// Creates new `EthcoreClient` with default `Fetch`. pub fn new( client: &Arc, miner: &Arc, @@ -60,6 +67,26 @@ impl EthcoreClient where C: MiningBlockChainClient, M: settings: Arc, signer: Option> ) -> Self { + Self::with_fetch(client, miner, sync, net, logger, settings, signer) + } +} + +impl EthcoreClient where + C: MiningBlockChainClient, + M: MinerService, + S: SyncProvider, + F: Fetch, { + + /// Creates new `EthcoreClient` with customizable `Fetch`. + pub fn with_fetch( + client: &Arc, + miner: &Arc, + sync: &Arc, + net: &Arc, + logger: Arc, + settings: Arc, + signer: Option> + ) -> Self { EthcoreClient { client: Arc::downgrade(client), miner: Arc::downgrade(miner), @@ -68,6 +95,7 @@ impl EthcoreClient where C: MiningBlockChainClient, M: logger: logger, settings: settings, signer: signer, + fetch: Mutex::new(F::default()), } } @@ -78,7 +106,11 @@ impl EthcoreClient where C: MiningBlockChainClient, M: } } -impl Ethcore for EthcoreClient where M: MinerService + 'static, C: MiningBlockChainClient + 'static, S: SyncProvider + 'static { +impl Ethcore for EthcoreClient where + M: MinerService + 'static, + C: MiningBlockChainClient + 'static, + S: SyncProvider + 'static, + F: Fetch + 'static { fn transactions_limit(&self, params: Params) -> Result { try!(self.active()); @@ -233,4 +265,42 @@ impl Ethcore for EthcoreClient where M: MinerService + Ok(to_value(&take_weak!(self.miner).all_transactions().into_iter().map(Into::into).collect::>())) } + + fn hash_content(&self, params: Params, ready: Ready) { + let res = self.active().and_then(|_| from_params::<(String,)>(params)); + + let hash_content = |result| { + let path = try!(result); + let mut file = io::BufReader::new(try!(fs::File::open(&path))); + // Try to hash + let result = sha3(&mut file); + // Remove file (always) + try!(fs::remove_file(&path)); + // Return the result + Ok(try!(result)) + }; + + match res { + Err(e) => ready.ready(Err(e)), + Ok((url, )) => { + let (tx, rx) = mpsc::channel(); + let res = self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| { + let result = hash_content(result) + .map_err(errors::from_fetch_error) + .map(|hash| to_value(H256::from(hash))); + + // Receive ready and invoke with result. + let ready: Ready = rx.try_recv().expect("When on_done is invoked ready object is always sent."); + ready.ready(result); + })); + + // Either invoke ready right away or transfer it to the closure. + if let Err(e) = res { + ready.ready(Err(errors::from_fetch_error(e))); + } else { + tx.send(ready).expect("Rx end is sent to on_done closure."); + } + } + } + } } diff --git a/rpc/src/v1/tests/helpers/fetch.rs b/rpc/src/v1/tests/helpers/fetch.rs new file mode 100644 index 000000000..98d888a10 --- /dev/null +++ b/rpc/src/v1/tests/helpers/fetch.rs @@ -0,0 +1,44 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Test implementation of fetch client. + +use std::io::Write; +use std::{env, fs, thread}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use fetch::{Fetch, FetchError, FetchResult}; + +/// Test implementation of fetcher. Will always return the same file. +#[derive(Default)] +pub struct TestFetch; + +impl Fetch for TestFetch { + fn request_async(&mut self, _url: &str, _abort: Arc, on_done: Box) -> Result<(), FetchError> { + thread::spawn(move || { + let mut path = env::temp_dir(); + path.push(Self::random_filename()); + + let mut file = fs::File::create(&path).unwrap(); + file.write_all(b"Some content").unwrap(); + + on_done(Ok(path)); + }); + Ok(()) + } +} + + diff --git a/rpc/src/v1/tests/helpers/mod.rs b/rpc/src/v1/tests/helpers/mod.rs index 1b8f9e256..234bae1be 100644 --- a/rpc/src/v1/tests/helpers/mod.rs +++ b/rpc/src/v1/tests/helpers/mod.rs @@ -18,6 +18,8 @@ mod sync_provider; mod miner_service; +mod fetch; pub use self::sync_provider::{Config, TestSyncProvider}; pub use self::miner_service::TestMinerService; +pub use self::fetch::TestFetch; diff --git a/rpc/src/v1/tests/mocked/ethcore.rs b/rpc/src/v1/tests/mocked/ethcore.rs index 811ccced4..3dc02e929 100644 --- a/rpc/src/v1/tests/mocked/ethcore.rs +++ b/rpc/src/v1/tests/mocked/ethcore.rs @@ -23,7 +23,7 @@ use ethcore::client::{TestBlockChainClient}; use jsonrpc_core::IoHandler; use v1::{Ethcore, EthcoreClient}; use v1::helpers::{SignerService, NetworkSettings}; -use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService}; +use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService, TestFetch}; use super::manage_network::TestManageNetwork; fn miner_service() -> Arc { @@ -60,12 +60,15 @@ fn network_service() -> Arc { Arc::new(TestManageNetwork) } +type TestEthcoreClient = EthcoreClient; + fn ethcore_client( client: &Arc, miner: &Arc, sync: &Arc, - net: &Arc) -> EthcoreClient { - EthcoreClient::new(client, miner, sync, net, logger(), settings(), None) + net: &Arc) + -> TestEthcoreClient { + EthcoreClient::with_fetch(client, miner, sync, net, logger(), settings(), None) } #[test] @@ -140,9 +143,9 @@ fn rpc_ethcore_dev_logs() { let logger = logger(); logger.append("a".to_owned()); logger.append("b".to_owned()); - let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger.clone(), settings(), None).to_delegate(); + let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger.clone(), settings(), None); let io = IoHandler::new(); - io.add_delegate(ethcore); + io.add_delegate(ethcore.to_delegate()); let request = r#"{"jsonrpc": "2.0", "method": "ethcore_devLogs", "params":[], "id": 1}"#; let response = r#"{"jsonrpc":"2.0","result":["b","a"],"id":1}"#; @@ -263,8 +266,8 @@ fn rpc_ethcore_unsigned_transactions_count() { let net = network_service(); let io = IoHandler::new(); let signer = Arc::new(SignerService::new_test()); - let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger(), settings(), Some(signer)).to_delegate(); - io.add_delegate(ethcore); + let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger(), settings(), Some(signer)); + io.add_delegate(ethcore.to_delegate()); let request = r#"{"jsonrpc": "2.0", "method": "ethcore_unsignedTransactionsCount", "params":[], "id": 1}"#; let response = r#"{"jsonrpc":"2.0","result":0,"id":1}"#; @@ -287,6 +290,21 @@ fn rpc_ethcore_unsigned_transactions_count_when_signer_disabled() { assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); } +#[test] +fn rpc_ethcore_hash_content() { + let miner = miner_service(); + let client = client_service(); + let sync = sync_provider(); + let net = network_service(); + let io = IoHandler::new(); + io.add_delegate(ethcore_client(&client, &miner, &sync, &net).to_delegate()); + + let request = r#"{"jsonrpc": "2.0", "method": "ethcore_hashContent", "params":["https://ethcore.io/assets/images/ethcore-black-horizontal.png"], "id": 1}"#; + let response = r#"{"jsonrpc":"2.0","result":"0x2be00befcf008bc0e7d9cdefc194db9c75352e8632f48498b5a6bfce9f02c88e","id":1}"#; + + assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); +} + #[test] fn rpc_ethcore_pending_transactions() { let miner = miner_service(); diff --git a/rpc/src/v1/traits/ethcore.rs b/rpc/src/v1/traits/ethcore.rs index 56c27534a..0565da04a 100644 --- a/rpc/src/v1/traits/ethcore.rs +++ b/rpc/src/v1/traits/ethcore.rs @@ -83,6 +83,9 @@ pub trait Ethcore: Sized + Send + Sync + 'static { /// Returns all pending (current) transactions from transaction queue. fn pending_transactions(&self, _: Params) -> Result; + /// Hash a file content under given URL. + fn hash_content(&self, _: Params, _: Ready); + /// Should be used to convert object to io delegate. fn to_delegate(self) -> IoDelegate { let mut delegate = IoDelegate::new(Arc::new(self)); @@ -107,6 +110,8 @@ pub trait Ethcore: Sized + Send + Sync + 'static { delegate.add_method("ethcore_registryAddress", Ethcore::registry_address); delegate.add_method("ethcore_encryptMessage", Ethcore::encrypt_message); delegate.add_method("ethcore_pendingTransactions", Ethcore::pending_transactions); + delegate.add_async_method("ethcore_hashContent", Ethcore::hash_content); + delegate } } diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml new file mode 100644 index 000000000..663d167bf --- /dev/null +++ b/util/fetch/Cargo.toml @@ -0,0 +1,18 @@ +[package] +description = "HTTP/HTTPS fetching library" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "fetch" +version = "0.1.0" +authors = ["Ethcore "] + +[dependencies] +log = "0.3" +rand = "0.3" +hyper = { default-features = false, git = "https://github.com/ethcore/hyper" } +https-fetch = { path = "../https-fetch" } +clippy = { version = "0.0.90", optional = true} + +[features] +default = [] +dev = ["clippy"] diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs new file mode 100644 index 000000000..bb8842a5b --- /dev/null +++ b/util/fetch/src/client.rs @@ -0,0 +1,146 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Fetching + +use std::{env, io}; +use std::sync::{mpsc, Arc}; +use std::sync::atomic::AtomicBool; +use std::path::PathBuf; + +use hyper; +use https_fetch as https; + +use fetch_file::{FetchHandler, Error as HttpFetchError}; + +pub type FetchResult = Result; + +#[derive(Debug)] +pub enum FetchError { + InvalidUrl, + Http(HttpFetchError), + Https(https::FetchError), + Io(io::Error), + Other(String), +} + +impl From for FetchError { + fn from(e: HttpFetchError) -> Self { + FetchError::Http(e) + } +} + +impl From for FetchError { + fn from(e: io::Error) -> Self { + FetchError::Io(e) + } +} + +pub trait Fetch: Default + Send { + /// Fetch URL and get the result in callback. + fn request_async(&mut self, url: &str, abort: Arc, on_done: Box) -> Result<(), FetchError>; + + /// Fetch URL and get a result Receiver. You will be notified when receiver is ready by `on_done` callback. + fn request(&mut self, url: &str, abort: Arc, on_done: Box) -> Result, FetchError> { + let (tx, rx) = mpsc::channel(); + try!(self.request_async(url, abort, Box::new(move |result| { + let res = tx.send(result); + if let Err(_) = res { + warn!("Fetch finished, but no one was listening"); + } + on_done(); + }))); + Ok(rx) + } + + /// Closes this client + fn close(self) {} + + /// Returns a random filename + fn random_filename() -> String { + use ::rand::Rng; + let mut rng = ::rand::OsRng::new().unwrap(); + rng.gen_ascii_chars().take(12).collect() + } +} + +pub struct Client { + http_client: hyper::Client, + https_client: https::Client, + limit: Option, +} + +impl Default for Client { + fn default() -> Self { + // Max 15MB will be downloaded. + Client::with_limit(Some(15*1024*1024)) + } +} + +impl Client { + fn with_limit(limit: Option) -> Self { + Client { + http_client: hyper::Client::new().expect("Unable to initialize http client."), + https_client: https::Client::with_limit(limit).expect("Unable to initialize https client."), + limit: limit, + } + } + + fn convert_url(url: hyper::Url) -> Result { + let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl))); + let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl)); + https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl) + } + + fn temp_path() -> PathBuf { + let mut dir = env::temp_dir(); + dir.push(Self::random_filename()); + dir + } +} + +impl Fetch for Client { + fn close(self) { + self.http_client.close(); + self.https_client.close(); + } + + fn request_async(&mut self, url: &str, abort: Arc, on_done: Box) -> Result<(), FetchError> { + let is_https = url.starts_with("https://"); + let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl)); + let temp_path = Self::temp_path(); + + trace!(target: "fetch", "Fetching from: {:?}", url); + + if is_https { + let url = try!(Self::convert_url(url)); + try!(self.https_client.fetch_to_file( + url, + temp_path.clone(), + abort, + move |result| on_done(result.map(|_| temp_path).map_err(FetchError::Https)), + ).map_err(|e| FetchError::Other(format!("{:?}", e)))); + } else { + try!(self.http_client.request( + url, + FetchHandler::new(temp_path, abort, Box::new(move |result| on_done(result)), self.limit.map(|v| v as u64).clone()), + ).map_err(|e| FetchError::Other(format!("{:?}", e)))); + } + + Ok(()) + } +} + diff --git a/dapps/src/handlers/client/fetch_file.rs b/util/fetch/src/fetch_file.rs similarity index 82% rename from dapps/src/handlers/client/fetch_file.rs rename to util/fetch/src/fetch_file.rs index c18fb6d5b..4801cc969 100644 --- a/dapps/src/handlers/client/fetch_file.rs +++ b/util/fetch/src/fetch_file.rs @@ -16,12 +16,11 @@ //! Hyper Client Handler to Fetch File -use std::{env, io, fs, fmt}; +use std::{io, fs, fmt}; use std::path::PathBuf; -use std::sync::{mpsc, Arc}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use random_filename; use hyper::status::StatusCode; use hyper::client::{Request, Response, DefaultTransport as HttpStream}; @@ -34,30 +33,31 @@ use super::FetchError; pub enum Error { Aborted, NotStarted, + SizeLimit, UnexpectedStatus(StatusCode), IoError(io::Error), HyperError(hyper::Error), } pub type FetchResult = Result; -pub type OnDone = Box; +pub type OnDone = Box; -pub struct Fetch { +pub struct FetchHandler { path: PathBuf, abort: Arc, file: Option, result: Option, - sender: mpsc::Sender, on_done: Option, + size_limit: Option, } -impl fmt::Debug for Fetch { +impl fmt::Debug for FetchHandler { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result) } } -impl Drop for Fetch { +impl Drop for FetchHandler { fn drop(&mut self) { let res = self.result.take().unwrap_or(Err(Error::NotStarted.into())); // Remove file if there was an error @@ -69,40 +69,35 @@ impl Drop for Fetch { } } // send result - let _ = self.sender.send(res); if let Some(f) = self.on_done.take() { - f(); + f(res); } } } -impl Fetch { - pub fn new(sender: mpsc::Sender, abort: Arc, on_done: OnDone) -> Self { - let mut dir = env::temp_dir(); - dir.push(random_filename()); - - Fetch { - path: dir, +impl FetchHandler { + pub fn new(path: PathBuf, abort: Arc, on_done: OnDone, size_limit: Option) -> Self { + FetchHandler { + path: path, abort: abort, file: None, result: None, - sender: sender, on_done: Some(on_done), + size_limit: size_limit, } } -} -impl Fetch { fn is_aborted(&self) -> bool { self.abort.load(Ordering::SeqCst) } + fn mark_aborted(&mut self) -> Next { self.result = Some(Err(Error::Aborted.into())); Next::end() } } -impl hyper::client::Handler for Fetch { +impl hyper::client::Handler for FetchHandler { fn on_request(&mut self, req: &mut Request) -> Next { if self.is_aborted() { return self.mark_aborted(); @@ -147,7 +142,19 @@ impl hyper::client::Handler for Fetch { } match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) { Ok(0) => Next::end(), - Ok(_) => read(), + Ok(bytes_read) => match self.size_limit { + None => read(), + // Check limit + Some(limit) if limit > bytes_read => { + self.size_limit = Some(limit - bytes_read); + read() + }, + // Size limit reached + _ => { + self.result = Some(Err(Error::SizeLimit.into())); + Next::end() + }, + }, Err(e) => match e.kind() { io::ErrorKind::WouldBlock => Next::read(), _ => { diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs new file mode 100644 index 000000000..8ec9e0ddd --- /dev/null +++ b/util/fetch/src/lib.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! A service to fetch any HTTP / HTTPS content. + +#[macro_use] +extern crate log; +extern crate hyper; +extern crate https_fetch; +extern crate rand; + + +pub mod client; +pub mod fetch_file; + +pub use self::client::{Client, Fetch, FetchError, FetchResult}; diff --git a/util/https-fetch/src/client.rs b/util/https-fetch/src/client.rs index 3e5d50515..ad75f2ca4 100644 --- a/util/https-fetch/src/client.rs +++ b/util/https-fetch/src/client.rs @@ -78,6 +78,10 @@ impl Drop for Client { impl Client { pub fn new() -> Result { + Self::with_limit(None) + } + + pub fn with_limit(size_limit: Option) -> Result { let mut event_loop = try!(mio::EventLoop::new()); let channel = event_loop.channel(); @@ -85,6 +89,7 @@ impl Client { let mut client = ClientLoop { next_token: 0, sessions: HashMap::new(), + size_limit: size_limit, }; event_loop.run(&mut client).unwrap(); }); @@ -128,6 +133,7 @@ impl Client { pub struct ClientLoop { next_token: usize, sessions: HashMap, + size_limit: Option, } impl mio::Handler for ClientLoop { @@ -154,7 +160,7 @@ impl mio::Handler for ClientLoop { let token = self.next_token; self.next_token += 1; - if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback) { + if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback, self.size_limit.clone()) { let httpreq = format!( "GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n", url.path(), diff --git a/util/https-fetch/src/http.rs b/util/https-fetch/src/http.rs index d29974c01..5f40ca4cb 100644 --- a/util/https-fetch/src/http.rs +++ b/util/https-fetch/src/http.rs @@ -35,18 +35,20 @@ pub struct HttpProcessor { status: Option, headers: Vec, body_writer: io::BufWriter>, + size_limit: Option, } const BREAK_LEN: usize = 2; impl HttpProcessor { - pub fn new(body_writer: Box) -> Self { + pub fn new(body_writer: Box, size_limit: Option) -> Self { HttpProcessor { state: State::WaitingForStatus, buffer: Cursor::new(Vec::new()), status: None, headers: Vec::new(), - body_writer: io::BufWriter::new(body_writer) + body_writer: io::BufWriter::new(body_writer), + size_limit: size_limit, } } @@ -140,6 +142,15 @@ impl HttpProcessor { }, State::WritingBody => { let len = self.buffer.get_ref().len(); + match self.size_limit { + None => {}, + Some(limit) if limit > len => {}, + _ => { + warn!("Finishing file fetching because limit was reached."); + self.set_state(State::Finished); + continue; + } + } try!(self.body_writer.write_all(self.buffer.get_ref())); self.buffer_consume(len); return Ok(()); @@ -167,6 +178,17 @@ impl HttpProcessor { }, // Buffers the data until we have a full chunk State::WritingChunk(left) if self.buffer.get_ref().len() >= left => { + match self.size_limit { + None => {}, + Some(limit) if limit > left => { + self.size_limit = Some(limit - left); + }, + _ => { + warn!("Finishing file fetching because limit was reached."); + self.set_state(State::Finished); + continue; + } + } try!(self.body_writer.write_all(&self.buffer.get_ref()[0..left])); self.buffer_consume(left + BREAK_LEN); @@ -230,7 +252,7 @@ mod tests { #[test] fn should_be_able_to_process_status_line() { // given - let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new()))); + let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None); // when let out = @@ -249,7 +271,7 @@ mod tests { #[test] fn should_be_able_to_process_headers() { // given - let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new()))); + let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None); // when let out = @@ -274,7 +296,7 @@ mod tests { fn should_be_able_to_consume_body() { // given let (writer, data) = Writer::new(); - let mut http = HttpProcessor::new(Box::new(writer)); + let mut http = HttpProcessor::new(Box::new(writer), None); // when let out = @@ -301,7 +323,7 @@ mod tests { fn should_correctly_handle_chunked_content() { // given let (writer, data) = Writer::new(); - let mut http = HttpProcessor::new(Box::new(writer)); + let mut http = HttpProcessor::new(Box::new(writer), None); // when let out = @@ -331,4 +353,40 @@ mod tests { assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]); assert_eq!(http.state(), State::Finished); } + + #[test] + fn should_stop_fetching_when_limit_is_reached() { + // given + let (writer, data) = Writer::new(); + let mut http = HttpProcessor::new(Box::new(writer), Some(5)); + + // when + let out = + "\ + HTTP/1.1 200 OK\r\n\ + Host: 127.0.0.1:8080\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: close\r\n\ + \r\n\ + 4\r\n\ + Pari\r\n\ + 3\r\n\ + ty \r\n\ + D\r\n\ + in\r\n\ + \r\n\ + chunks.\r\n\ + 0\r\n\ + \r\n\ + "; + http.write_all(out.as_bytes()).unwrap(); + http.flush().unwrap(); + + // then + assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK"); + assert_eq!(http.headers().len(), 3); + assert_eq!(data.borrow().get_ref()[..], b"Pari"[..]); + assert_eq!(http.state(), State::Finished); + } + } diff --git a/util/https-fetch/src/tlsclient.rs b/util/https-fetch/src/tlsclient.rs index e3ce44764..62af2b06c 100644 --- a/util/https-fetch/src/tlsclient.rs +++ b/util/https-fetch/src/tlsclient.rs @@ -87,6 +87,7 @@ impl TlsClient { writer: Box, abort: Arc, mut callback: Box, + size_limit: Option, ) -> Result { let res = TlsClient::make_config().and_then(|cfg| { TcpStream::connect(url.address()).map(|sock| { @@ -98,7 +99,7 @@ impl TlsClient { Ok((cfg, sock)) => Ok(TlsClient { abort: abort, token: token, - writer: HttpProcessor::new(writer), + writer: HttpProcessor::new(writer, size_limit), socket: sock, closing: false, error: None,