Restarting fetch client every now and then (#4399)
This commit is contained in:
Tomasz Drwięga
Gav Wood
parent
4961422523
commit
015425d447
94
Cargo.lock
generated
94
Cargo.lock
generated
@ -73,6 +73,11 @@ name = "ansi_term"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "antidote"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "app_dirs"
|
||||
version = "1.1.1"
|
||||
@ -285,6 +290,11 @@ name = "dtoa"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "elastic-array"
|
||||
version = "0.6.0"
|
||||
@ -778,7 +788,8 @@ dependencies = [
|
||||
"futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -895,6 +906,35 @@ dependencies = [
|
||||
"vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-native-tls"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.1.0"
|
||||
@ -948,6 +988,11 @@ name = "itoa"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "jsonrpc-core"
|
||||
version = "5.0.0"
|
||||
@ -1706,15 +1751,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.2.0"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hyper-native-tls 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_urlencoded 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@ -1900,6 +1945,11 @@ name = "serde"
|
||||
version = "0.8.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde_codegen"
|
||||
version = "0.8.19"
|
||||
@ -1932,11 +1982,24 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.3.0"
|
||||
name = "serde_json"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@ -2396,6 +2459,7 @@ dependencies = [
|
||||
"checksum advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a"
|
||||
"checksum aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67077478f0a03952bed2e6786338d400d40c25e9836e08ad50af96607317fd03"
|
||||
"checksum ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f46cd5b1d660c938e3f92dfe7a73d832b3281479363dd0cd9c1c2fbf60f7962"
|
||||
"checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5"
|
||||
"checksum app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7d1c0d48a81bbb13043847f957971f4d87c81542d80ece5e84ba3cba4058fd4"
|
||||
"checksum arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "16e3bdb2f54b3ace0285975d59a97cf8ed3855294b2b6bc651fcf22a9c352975"
|
||||
"checksum aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07d344974f0a155f091948aa389fb1b912d3a58414fbdb9c8d446d193ee3496a"
|
||||
@ -2424,6 +2488,7 @@ dependencies = [
|
||||
"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf"
|
||||
"checksum docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc0acb4ce0828c6a5a11d47baa432fe885881c27428c3a4e473e454ffe57a76"
|
||||
"checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d"
|
||||
"checksum dtoa 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5edd69c67b2f8e0911629b7e6b8a34cb3956613cd7c6e6414966dee349c2db4f"
|
||||
"checksum elastic-array 0.6.0 (git+https://github.com/ethcore/elastic-array)" = "<none>"
|
||||
"checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5"
|
||||
"checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
|
||||
@ -2440,12 +2505,15 @@ dependencies = [
|
||||
"checksum hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2da7d3a34cf6406d9d700111b8eafafe9a251de41ae71d8052748259343b58"
|
||||
"checksum httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46534074dbb80b070d60a5cb8ecadd8963a00a438ae1a95268850a7ef73b67ae"
|
||||
"checksum hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)" = "<none>"
|
||||
"checksum hyper 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "220407e5a263f110ec30a071787c9535918fdfc97def5680c90013c3f30c38c1"
|
||||
"checksum hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bcb3fc65554155980167fb821d05c7c66177f92464976c0b676a19d9e03387a7"
|
||||
"checksum hyper-native-tls 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "afe68f772f0497a7205e751626bb8e1718568b58534b6108c73a74ef80483409"
|
||||
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
|
||||
"checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484"
|
||||
"checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c"
|
||||
"checksum itertools 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "086e1fa5fe48840b1cfdef3a20c7e3115599f8d5c4c87ef32a794a7cdd184d76"
|
||||
"checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
|
||||
"checksum itoa 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91fd9dc2c587067de817fec4ad355e3818c3d893a78cab32a0a474c7a15bb8d5"
|
||||
"checksum jsonrpc-core 5.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "<none>"
|
||||
"checksum jsonrpc-http-server 7.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "<none>"
|
||||
"checksum jsonrpc-ipc-server 1.0.0 (git+https://github.com/ethcore/jsonrpc.git)" = "<none>"
|
||||
@ -2520,7 +2588,7 @@ dependencies = [
|
||||
"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea"
|
||||
"checksum regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)" = "b4329b8928a284580a1c63ec9d846b12f6d3472317243ff7077aff11f23f2b29"
|
||||
"checksum regex-syntax 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "841591b1e05609a643e3b4d0045fce04f701daba7151ddcd3ad47b080693d5a9"
|
||||
"checksum reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83186fee0d4dbeb95e610b77b05b05cf5b31703dd375222acb74c3dff4be957c"
|
||||
"checksum reqwest 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bef9ed8fdfcc30947d6b774938dc0c3f369a474efe440df2c7f278180b2d2e6"
|
||||
"checksum rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
|
||||
"checksum rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
|
||||
"checksum rotor 0.6.3 (git+https://github.com/ethcore/rotor)" = "<none>"
|
||||
@ -2539,10 +2607,12 @@ dependencies = [
|
||||
"checksum semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae2ff60ecdb19c255841c066cbfa5f8c2a4ada1eb3ae47c77ab6667128da71f5"
|
||||
"checksum semver-parser 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e88e43a5a74dd2a11707f9c21dfd4a423c66bd871df813227bb0a3e78f3a1ae9"
|
||||
"checksum serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)" = "58a19c0871c298847e6b68318484685cd51fa5478c0c905095647540031356e5"
|
||||
"checksum serde 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4d8f810025e9d09c4eaa49c16eaf878f34a947889e878cd7d3b5bef3197cc119"
|
||||
"checksum serde_codegen 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)" = "ce29a6ae259579707650ec292199b5fed2c0b8e2a4bdc994452d24d1bcf2242a"
|
||||
"checksum serde_codegen_internals 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "59933a62554548c690d2673c5164f0c4a46be7c5731edfd94b0ecb1048940732"
|
||||
"checksum serde_json 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3f7d3c184d35801fb8b32b46a7d58d57dbcc150b0eb2b46a1eb79645e8ecfd5b"
|
||||
"checksum serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "53d4ebaa8d1d4f90d1b63dfca81ccd98ac20e1e479dbae393cbaf60f6fecd8d8"
|
||||
"checksum serde_json 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fea48f4d4df4e620e3c81fd2bf28c93dd0d266361a76bac4f254b71f0e13f3cd"
|
||||
"checksum serde_urlencoded 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a81f15da4b9780e1524697f73b09076b6e42298ef673bead9ca8f848b334ef84"
|
||||
"checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c"
|
||||
"checksum shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72f20b8f3c060374edb8046591ba28f62448c369ccbdc7b02075103fb3a9e38d"
|
||||
"checksum siphasher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c44e42fa187b5a8782489cf7740cc27c3125806be2bf33563cf5e02e9533fcd"
|
||||
|
||||
@ -9,8 +9,9 @@ authors = ["Parity Technologies <admin@parity.io>"]
|
||||
[dependencies]
|
||||
futures = "0.1"
|
||||
futures-cpupool = "0.1"
|
||||
parking_lot = "0.3"
|
||||
log = "0.3"
|
||||
reqwest = "0.2"
|
||||
reqwest = "0.4"
|
||||
mime = "0.2"
|
||||
clippy = { version = "0.0.90", optional = true}
|
||||
|
||||
|
||||
@ -16,13 +16,14 @@
|
||||
|
||||
//! Fetching
|
||||
|
||||
use std::{io, fmt};
|
||||
use std::{io, fmt, time};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{self, AtomicBool};
|
||||
|
||||
use futures::{self, BoxFuture, Future};
|
||||
use futures_cpupool::{CpuPool, CpuFuture};
|
||||
use mime::{self, Mime};
|
||||
use parking_lot::RwLock;
|
||||
use reqwest;
|
||||
|
||||
#[derive(Default, Debug, Clone)]
|
||||
@ -73,24 +74,52 @@ pub trait Fetch: Clone + Send + Sync + 'static {
|
||||
fn close(self) where Self: Sized {}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
const CLIENT_TIMEOUT_SECONDS: u64 = 5;
|
||||
|
||||
pub struct Client {
|
||||
client: Arc<reqwest::Client>,
|
||||
client: RwLock<(time::Instant, Arc<reqwest::Client>)>,
|
||||
pool: CpuPool,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Clone for Client {
|
||||
fn clone(&self) -> Self {
|
||||
let (ref time, ref client) = *self.client.read();
|
||||
Client {
|
||||
client: RwLock::new((time.clone(), client.clone())),
|
||||
pool: self.pool.clone(),
|
||||
limit: self.limit.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn with_limit(limit: Option<usize>) -> Result<Self, Error> {
|
||||
fn new_client() -> Result<Arc<reqwest::Client>, Error> {
|
||||
let mut client = reqwest::Client::new()?;
|
||||
client.redirect(reqwest::RedirectPolicy::limited(5));
|
||||
Ok(Arc::new(client))
|
||||
}
|
||||
|
||||
fn with_limit(limit: Option<usize>) -> Result<Self, Error> {
|
||||
Ok(Client {
|
||||
client: Arc::new(client),
|
||||
client: RwLock::new((time::Instant::now(), Self::new_client()?)),
|
||||
pool: CpuPool::new(4),
|
||||
limit: limit,
|
||||
})
|
||||
}
|
||||
|
||||
fn client(&self) -> Result<Arc<reqwest::Client>, Error> {
|
||||
{
|
||||
let (ref time, ref client) = *self.client.read();
|
||||
if time.elapsed() < time::Duration::from_secs(CLIENT_TIMEOUT_SECONDS) {
|
||||
return Ok(client.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let client = Self::new_client()?;
|
||||
*self.client.write() = (time::Instant::now(), client.clone());
|
||||
Ok(client)
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetch for Client {
|
||||
@ -112,12 +141,19 @@ impl Fetch for Client {
|
||||
fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result {
|
||||
debug!(target: "fetch", "Fetching from: {:?}", url);
|
||||
|
||||
self.pool.spawn(FetchTask {
|
||||
url: url.into(),
|
||||
client: self.client.clone(),
|
||||
limit: self.limit,
|
||||
abort: abort,
|
||||
})
|
||||
match self.client() {
|
||||
Ok(client) => {
|
||||
self.pool.spawn(FetchTask {
|
||||
url: url.into(),
|
||||
client: client,
|
||||
limit: self.limit,
|
||||
abort: abort,
|
||||
})
|
||||
},
|
||||
Err(err) => {
|
||||
self.pool.spawn(futures::future::err(err))
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -21,6 +21,7 @@ extern crate log;
|
||||
|
||||
extern crate futures;
|
||||
extern crate futures_cpupool;
|
||||
extern crate parking_lot;
|
||||
extern crate reqwest;
|
||||
|
||||
pub extern crate mime;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user