diff --git a/Cargo.lock b/Cargo.lock index 0955b4465..0642ebc92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,11 @@ name = "ansi_term" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "antidote" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "app_dirs" version = "1.1.1" @@ -285,6 +290,11 @@ name = "dtoa" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "dtoa" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "elastic-array" version = "0.6.0" @@ -778,7 +788,8 @@ dependencies = [ "futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -895,6 +906,35 @@ dependencies = [ "vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hyper" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-native-tls" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "idna" version = "0.1.0" @@ -948,6 +988,11 @@ name = "itoa" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "itoa" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "jsonrpc-core" version = "5.0.0" @@ -1706,15 +1751,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "reqwest" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-native-tls 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_urlencoded 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1900,6 +1945,11 @@ name = "serde" version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "serde" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde_codegen" version = "0.8.19" @@ -1932,11 +1982,24 @@ dependencies = [ ] [[package]] -name = "serde_urlencoded" -version = "0.3.0" +name = "serde_json" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)", + "dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_urlencoded" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2396,6 +2459,7 @@ dependencies = [ "checksum advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a" "checksum aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67077478f0a03952bed2e6786338d400d40c25e9836e08ad50af96607317fd03" "checksum ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f46cd5b1d660c938e3f92dfe7a73d832b3281479363dd0cd9c1c2fbf60f7962" +"checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" "checksum app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7d1c0d48a81bbb13043847f957971f4d87c81542d80ece5e84ba3cba4058fd4" "checksum arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "16e3bdb2f54b3ace0285975d59a97cf8ed3855294b2b6bc651fcf22a9c352975" "checksum aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07d344974f0a155f091948aa389fb1b912d3a58414fbdb9c8d446d193ee3496a" @@ -2424,6 +2488,7 @@ dependencies = [ "checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf" "checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76" "checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d" +"checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f" "checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "" "checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5" "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "" @@ -2440,12 +2505,15 @@ dependencies = [ "checksum hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2da7d3a34cf6406d9d700111b8eafafe9a251de41ae71d8052748259343b58" "checksum httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46534074dbb80b070d60a5cb8ecadd8963a00a438ae1a95268850a7ef73b67ae" "checksum hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)" = "" +"checksum hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "220407e5a263f110ec30a071787c9535918fdfc97def5680c90013c3f30c38c1" "checksum hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bcb3fc65554155980167fb821d05c7c66177f92464976c0b676a19d9e03387a7" +"checksum hyper-native-tls 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "afe68f772f0497a7205e751626bb8e1718568b58534b6108c73a74ef80483409" "checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11" "checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484" "checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c" "checksum itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "086e1fa5fe48840b1cfdef3a20c7e3115599f8d5c4c87ef32a794a7cdd184d76" "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1" +"checksum itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91fd9dc2c587067de817fec4ad355e3818c3d893a78cab32a0a474c7a15bb8d5" "checksum jsonrpc-core 5.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "" "checksum jsonrpc-http-server 7.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "" "checksum jsonrpc-ipc-server 1.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "" @@ -2520,7 +2588,7 @@ dependencies = [ "checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea" "checksum regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)" = "b4329b8928a284580a1c63ec9d846b12f6d3472317243ff7077aff11f23f2b29" "checksum regex-syntax 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "841591b1e05609a643e3b4d0045fce04f701daba7151ddcd3ad47b080693d5a9" -"checksum reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83186fee0d4dbeb95e610b77b05b05cf5b31703dd375222acb74c3dff4be957c" +"checksum reqwest 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bef9ed8fdfcc30947d6b774938dc0c3f369a474efe440df2c7f278180b2d2e6" "checksum rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)" = "" "checksum rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)" = "" "checksum rotor 0.6.3 (git+https://github.com/ethcore/rotor)" = "" @@ -2539,10 +2607,12 @@ dependencies = [ "checksum semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae2ff60ecdb19c255841c066cbfa5f8c2a4ada1eb3ae47c77ab6667128da71f5" "checksum semver-parser 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e88e43a5a74dd2a11707f9c21dfd4a423c66bd871df813227bb0a3e78f3a1ae9" "checksum serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)" = "58a19c0871c298847e6b68318484685cd51fa5478c0c905095647540031356e5" +"checksum serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4d8f810025e9d09c4eaa49c16eaf878f34a947889e878cd7d3b5bef3197cc119" "checksum serde_codegen 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)" = "ce29a6ae259579707650ec292199b5fed2c0b8e2a4bdc994452d24d1bcf2242a" "checksum serde_codegen_internals 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "59933a62554548c690d2673c5164f0c4a46be7c5731edfd94b0ecb1048940732" "checksum serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3f7d3c184d35801fb8b32b46a7d58d57dbcc150b0eb2b46a1eb79645e8ecfd5b" -"checksum serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "53d4ebaa8d1d4f90d1b63dfca81ccd98ac20e1e479dbae393cbaf60f6fecd8d8" +"checksum serde_json 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fea48f4d4df4e620e3c81fd2bf28c93dd0d266361a76bac4f254b71f0e13f3cd" +"checksum serde_urlencoded 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a81f15da4b9780e1524697f73b09076b6e42298ef673bead9ca8f848b334ef84" "checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" "checksum shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72f20b8f3c060374edb8046591ba28f62448c369ccbdc7b02075103fb3a9e38d" "checksum siphasher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c44e42fa187b5a8782489cf7740cc27c3125806be2bf33563cf5e02e9533fcd" diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index 308816ed7..96c1317f2 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -9,8 +9,9 @@ authors = ["Parity Technologies "] [dependencies] futures = "0.1" futures-cpupool = "0.1" +parking_lot = "0.3" log = "0.3" -reqwest = "0.2" +reqwest = "0.4" mime = "0.2" clippy = { version = "0.0.90", optional = true} diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 7bc2b211d..09fe4741b 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -16,13 +16,14 @@ //! Fetching -use std::{io, fmt}; +use std::{io, fmt, time}; use std::sync::Arc; use std::sync::atomic::{self, AtomicBool}; use futures::{self, BoxFuture, Future}; use futures_cpupool::{CpuPool, CpuFuture}; use mime::{self, Mime}; +use parking_lot::RwLock; use reqwest; #[derive(Default, Debug, Clone)] @@ -73,24 +74,52 @@ pub trait Fetch: Clone + Send + Sync + 'static { fn close(self) where Self: Sized {} } -#[derive(Clone)] +const CLIENT_TIMEOUT_SECONDS: u64 = 5; + pub struct Client { - client: Arc, + client: RwLock<(time::Instant, Arc)>, pool: CpuPool, limit: Option, } +impl Clone for Client { + fn clone(&self) -> Self { + let (ref time, ref client) = *self.client.read(); + Client { + client: RwLock::new((time.clone(), client.clone())), + pool: self.pool.clone(), + limit: self.limit.clone(), + } + } +} + impl Client { - fn with_limit(limit: Option) -> Result { + fn new_client() -> Result, Error> { let mut client = reqwest::Client::new()?; client.redirect(reqwest::RedirectPolicy::limited(5)); + Ok(Arc::new(client)) + } + fn with_limit(limit: Option) -> Result { Ok(Client { - client: Arc::new(client), + client: RwLock::new((time::Instant::now(), Self::new_client()?)), pool: CpuPool::new(4), limit: limit, }) } + + fn client(&self) -> Result, Error> { + { + let (ref time, ref client) = *self.client.read(); + if time.elapsed() < time::Duration::from_secs(CLIENT_TIMEOUT_SECONDS) { + return Ok(client.clone()); + } + } + + let client = Self::new_client()?; + *self.client.write() = (time::Instant::now(), client.clone()); + Ok(client) + } } impl Fetch for Client { @@ -112,12 +141,19 @@ impl Fetch for Client { fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result { debug!(target: "fetch", "Fetching from: {:?}", url); - self.pool.spawn(FetchTask { - url: url.into(), - client: self.client.clone(), - limit: self.limit, - abort: abort, - }) + match self.client() { + Ok(client) => { + self.pool.spawn(FetchTask { + url: url.into(), + client: client, + limit: self.limit, + abort: abort, + }) + }, + Err(err) => { + self.pool.spawn(futures::future::err(err)) + }, + } } } diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index a013a0bcb..34091f4bc 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -21,6 +21,7 @@ extern crate log; extern crate futures; extern crate futures_cpupool; +extern crate parking_lot; extern crate reqwest; pub extern crate mime;