// Copyright 2015-2018 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see .
use futures::future::{self, Loop};
use futures::sync::{mpsc, oneshot};
use futures::{self, Future, Async, Sink, Stream};
use hyper::header::{self, HeaderMap, HeaderValue, IntoHeaderName};
use hyper::{self, Method, StatusCode};
use hyper_rustls;
use std;
use std::cmp::min;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc::RecvTimeoutError;
use std::thread;
use std::time::Duration;
use std::{io, fmt};
use tokio::{self, util::FutureExt};
use url::{self, Url};
use bytes::Bytes;
const MAX_SIZE: usize = 64 * 1024 * 1024;
const MAX_SECS: Duration = Duration::from_secs(5);
const MAX_REDR: usize = 5;
/// A handle to abort requests.
///
/// Requests are either aborted based on reaching thresholds such as
/// maximum response size, timeouts or too many redirects, or else
/// they can be aborted explicitly by the calling code.
#[derive(Clone, Debug)]
pub struct Abort {
abort: Arc,
size: usize,
time: Duration,
redir: usize,
}
impl Default for Abort {
fn default() -> Abort {
Abort {
abort: Arc::new(AtomicBool::new(false)),
size: MAX_SIZE,
time: MAX_SECS,
redir: MAX_REDR,
}
}
}
impl From> for Abort {
fn from(a: Arc) -> Abort {
Abort {
abort: a,
size: MAX_SIZE,
time: MAX_SECS,
redir: MAX_REDR,
}
}
}
impl Abort {
/// True if `abort` has been invoked.
pub fn is_aborted(&self) -> bool {
self.abort.load(Ordering::SeqCst)
}
/// The maximum response body size.
pub fn max_size(&self) -> usize {
self.size
}
/// The maximum total time, including redirects.
pub fn max_duration(&self) -> Duration {
self.time
}
/// The maximum number of redirects to allow.
pub fn max_redirects(&self) -> usize {
self.redir
}
/// Mark as aborted.
pub fn abort(&self) {
self.abort.store(true, Ordering::SeqCst)
}
/// Set the maximum reponse body size.
pub fn with_max_size(self, n: usize) -> Abort {
Abort { size: n, .. self }
}
/// Set the maximum duration (including redirects).
pub fn with_max_duration(self, d: Duration) -> Abort {
Abort { time: d, .. self }
}
/// Set the maximum number of redirects to follow.
pub fn with_max_redirects(self, n: usize) -> Abort {
Abort { redir: n, .. self }
}
}
/// Types which retrieve content from some URL.
pub trait Fetch: Clone + Send + Sync + 'static {
/// The result future.
type Result: Future + Send + 'static;
/// Make a request to given URL
fn fetch(&self, request: Request, abort: Abort) -> Self::Result;
/// Get content from some URL.
fn get(&self, url: &str, abort: Abort) -> Self::Result;
/// Post content to some URL.
fn post(&self, url: &str, abort: Abort) -> Self::Result;
}
type TxResponse = oneshot::Sender>;
type TxStartup = std::sync::mpsc::SyncSender>;
type ChanItem = Option<(Request, Abort, TxResponse)>;
/// An implementation of `Fetch` using a `hyper` client.
// Due to the `Send` bound of `Fetch` we spawn a background thread for
// actual request/response processing as `hyper::Client` itself does
// not implement `Send` currently.
#[derive(Debug)]
pub struct Client {
runtime: mpsc::Sender,
refs: Arc,
}
// When cloning a client we increment the internal reference counter.
impl Clone for Client {
fn clone(&self) -> Client {
self.refs.fetch_add(1, Ordering::SeqCst);
Client {
runtime: self.runtime.clone(),
refs: self.refs.clone(),
}
}
}
// When dropping a client, we decrement the reference counter.
// Once it reaches 0 we terminate the background thread.
impl Drop for Client {
fn drop(&mut self) {
if self.refs.fetch_sub(1, Ordering::SeqCst) == 1 {
// ignore send error as it means the background thread is gone already
let _ = self.runtime.clone().send(None).wait();
}
}
}
impl Client {
/// Create a new fetch client.
pub fn new(num_dns_threads: usize) -> Result {
let (tx_start, rx_start) = std::sync::mpsc::sync_channel(1);
let (tx_proto, rx_proto) = mpsc::channel(64);
Client::background_thread(tx_start, rx_proto, num_dns_threads)?;
match rx_start.recv_timeout(Duration::from_secs(10)) {
Err(RecvTimeoutError::Timeout) => {
error!(target: "fetch", "timeout starting background thread");
return Err(Error::BackgroundThreadDead)
}
Err(RecvTimeoutError::Disconnected) => {
error!(target: "fetch", "background thread gone");
return Err(Error::BackgroundThreadDead)
}
Ok(Err(e)) => {
error!(target: "fetch", "error starting background thread: {}", e);
return Err(e.into())
}
Ok(Ok(())) => {}
}
Ok(Client {
runtime: tx_proto,
refs: Arc::new(AtomicUsize::new(1)),
})
}
fn background_thread(tx_start: TxStartup, rx_proto: mpsc::Receiver, num_dns_threads: usize) -> io::Result> {
thread::Builder::new().name("fetch".into()).spawn(move || {
let mut runtime = match tokio::runtime::current_thread::Runtime::new() {
Ok(c) => c,
Err(e) => return tx_start.send(Err(e)).unwrap_or(())
};
let hyper = hyper::Client::builder()
.build(hyper_rustls::HttpsConnector::new(num_dns_threads));
let future = rx_proto.take_while(|item| Ok(item.is_some()))
.map(|item| item.expect("`take_while` is only passing on channel items != None; qed"))
.for_each(|(request, abort, sender)|
{
trace!(target: "fetch", "new request to {}", request.url());
if abort.is_aborted() {
return future::ok(sender.send(Err(Error::Aborted)).unwrap_or(()))
}
let ini = (hyper.clone(), request, abort, 0);
let fut = future::loop_fn(ini, |(client, request, abort, redirects)| {
let request2 = request.clone();
let url2 = request2.url().clone();
let abort2 = abort.clone();
client.request(request.into())
.map(move |resp| Response::new(url2, resp, abort2))
.from_err()
.and_then(move |resp| {
if abort.is_aborted() {
debug!(target: "fetch", "fetch of {} aborted", request2.url());
return Err(Error::Aborted)
}
if let Some((next_url, preserve_method)) = redirect_location(request2.url().clone(), &resp) {
if redirects >= abort.max_redirects() {
return Err(Error::TooManyRedirects)
}
let request = if preserve_method {
let mut request2 = request2.clone();
request2.set_url(next_url);
request2
} else {
Request::new(next_url, Method::GET)
};
Ok(Loop::Continue((client, request, abort, redirects + 1)))
} else {
if let Some(ref h_val) = resp.headers.get(header::CONTENT_LENGTH) {
let content_len = h_val
.to_str()?
.parse::()?;
if content_len > abort.max_size() as u64 {
return Err(Error::SizeLimit)
}
}
Ok(Loop::Break(resp))
}
})
})
.then(|result| {
future::ok(sender.send(result).unwrap_or(()))
});
tokio::spawn(fut);
trace!(target: "fetch", "waiting for next request ...");
future::ok(())
});
tx_start.send(Ok(())).unwrap_or(());
debug!(target: "fetch", "processing requests ...");
if let Err(()) = runtime.block_on(future) {
error!(target: "fetch", "error while executing future")
}
debug!(target: "fetch", "fetch background thread finished")
})
}
}
impl Fetch for Client {
type Result = Box + Send + 'static>;
fn fetch(&self, request: Request, abort: Abort) -> Self::Result {
debug!(target: "fetch", "fetching: {:?}", request.url());
if abort.is_aborted() {
return Box::new(future::err(Error::Aborted))
}
let (tx_res, rx_res) = oneshot::channel();
let maxdur = abort.max_duration();
let sender = self.runtime.clone();
let future = sender.send(Some((request, abort, tx_res)))
.map_err(|e| {
error!(target: "fetch", "failed to schedule request: {}", e);
Error::BackgroundThreadDead
})
.and_then(|_| rx_res.map_err(|oneshot::Canceled| Error::BackgroundThreadDead))
.and_then(future::result);
Box::new(future.timeout(maxdur)
.map_err(|err| {
if err.is_inner() {
Error::from(err.into_inner().unwrap())
} else {
Error::from(err)
}
})
)
}
/// Get content from some URL.
fn get(&self, url: &str, abort: Abort) -> Self::Result {
let url: Url = match url.parse() {
Ok(u) => u,
Err(e) => return Box::new(future::err(e.into()))
};
self.fetch(Request::get(url), abort)
}
/// Post content to some URL.
fn post(&self, url: &str, abort: Abort) -> Self::Result {
let url: Url = match url.parse() {
Ok(u) => u,
Err(e) => return Box::new(future::err(e.into()))
};
self.fetch(Request::post(url), abort)
}
}
// Extract redirect location from response. The second return value indicate whether the original method should be preserved.
fn redirect_location(u: Url, r: &Response) -> Option<(Url, bool)> {
let preserve_method = match r.status() {
StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT => true,
_ => false,
};
match r.status() {
StatusCode::MOVED_PERMANENTLY
| StatusCode::PERMANENT_REDIRECT
| StatusCode::TEMPORARY_REDIRECT
| StatusCode::FOUND
| StatusCode::SEE_OTHER => {
r.headers.get(header::LOCATION).and_then(|loc| {
loc.to_str().ok().and_then(|loc_s| {
u.join(loc_s).ok().map(|url| (url, preserve_method))
})
})
}
_ => None
}
}
/// A wrapper for hyper::Request using Url and with methods.
#[derive(Debug, Clone)]
pub struct Request {
url: Url,
method: Method,
headers: HeaderMap,
body: Bytes,
}
impl Request {
/// Create a new request, with given url and method.
pub fn new(url: Url, method: Method) -> Request {
Request {
url, method,
headers: HeaderMap::new(),
body: Default::default(),
}
}
/// Create a new GET request.
pub fn get(url: Url) -> Request {
Request::new(url, Method::GET)
}
/// Create a new empty POST request.
pub fn post(url: Url) -> Request {
Request::new(url, Method::POST)
}
/// Read the url.
pub fn url(&self) -> &Url {
&self.url
}
/// Read the request headers.
pub fn headers(&self) -> &HeaderMap {
&self.headers
}
/// Get a mutable reference to the headers.
pub fn headers_mut(&mut self) -> &mut HeaderMap {
&mut self.headers
}
/// Set the body of the request.
pub fn set_body>(&mut self, body: T) {
self.body = body.into();
}
/// Set the url of the request.
pub fn set_url(&mut self, url: Url) {
self.url = url;
}
/// Consume self, and return it with the added given header.
pub fn with_header(mut self, key: K, val: HeaderValue) -> Self
where K: IntoHeaderName,
{
self.headers_mut().append(key, val);
self
}
/// Consume self, and return it with the body.
pub fn with_body>(mut self, body: T) -> Self {
self.set_body(body);
self
}
}
impl From for hyper::Request {
fn from(req: Request) -> hyper::Request {
let uri: hyper::Uri = req.url.as_ref().parse().expect("Every valid URLis also a URI.");
hyper::Request::builder()
.method(req.method)
.uri(uri)
.header(header::USER_AGENT, HeaderValue::from_static("Parity Fetch Neo"))
.body(req.body.into())
.expect("Header, uri, method, and body are already valid and can not fail to parse; qed")
}
}
/// An HTTP response.
#[derive(Debug)]
pub struct Response {
url: Url,
status: StatusCode,
headers: HeaderMap,
body: hyper::Body,
abort: Abort,
nread: usize,
}
impl Response {
/// Create a new response, wrapping a hyper response.
pub fn new(u: Url, r: hyper::Response, a: Abort) -> Response {
Response {
url: u,
status: r.status(),
headers: r.headers().clone(),
body: r.into_body(),
abort: a,
nread: 0,
}
}
/// The response status.
pub fn status(&self) -> StatusCode {
self.status
}
/// Status code == OK (200)?
pub fn is_success(&self) -> bool {
self.status() == StatusCode::OK
}
/// Status code == 404.
pub fn is_not_found(&self) -> bool {
self.status() == StatusCode::NOT_FOUND
}
/// Is the content-type text/html?
pub fn is_html(&self) -> bool {
self.headers.get(header::CONTENT_TYPE).and_then(|ct_val| {
ct_val.to_str().ok().map(|ct_str| {
ct_str.contains("text") && ct_str.contains("html")
})
}).unwrap_or(false)
}
}
impl Stream for Response {
type Item = hyper::Chunk;
type Error = Error;
fn poll(&mut self) -> futures::Poll