Merge branch 'master' into dapps-https

Conflicts:
	Cargo.lock
	dapps/src/handlers/fetch.rs
This commit is contained in:
Tomasz Drwięga
2016-08-31 11:29:23 +02:00
45 changed files with 865 additions and 448 deletions

View File

@@ -22,6 +22,7 @@ serde_json = "0.7.0"
serde_macros = { version = "0.7.0", optional = true }
zip = { version = "0.1", default-features = false }
ethabi = "0.2.1"
linked-hash-map = "0.3"
ethcore-rpc = { path = "../rpc" }
ethcore-util = { path = "../util" }
https-fetch = { path = "../util/https-fetch" }

128
dapps/src/apps/cache.rs Normal file
View File

@@ -0,0 +1,128 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Fetchable Dapps support.
use std::fs;
use std::sync::{Arc};
use std::sync::atomic::{AtomicBool, Ordering};
use linked_hash_map::LinkedHashMap;
use page::LocalPageEndpoint;
pub enum ContentStatus {
Fetching(Arc<AtomicBool>),
Ready(LocalPageEndpoint),
}
#[derive(Default)]
pub struct ContentCache {
cache: LinkedHashMap<String, ContentStatus>,
}
impl ContentCache {
pub fn insert(&mut self, content_id: String, status: ContentStatus) -> Option<ContentStatus> {
self.cache.insert(content_id, status)
}
pub fn remove(&mut self, content_id: &str) -> Option<ContentStatus> {
self.cache.remove(content_id)
}
pub fn get(&mut self, content_id: &str) -> Option<&mut ContentStatus> {
self.cache.get_refresh(content_id)
}
pub fn clear_garbage(&mut self, expected_size: usize) -> Vec<(String, ContentStatus)> {
let mut len = self.cache.len();
if len <= expected_size {
return Vec::new();
}
let mut removed = Vec::with_capacity(len - expected_size);
while len > expected_size {
let entry = self.cache.pop_front().unwrap();
match entry.1 {
ContentStatus::Fetching(ref abort) => {
trace!(target: "dapps", "Aborting {} because of limit.", entry.0);
// Mark as aborted
abort.store(true, Ordering::Relaxed);
},
ContentStatus::Ready(ref endpoint) => {
trace!(target: "dapps", "Removing {} because of limit.", entry.0);
// Remove path
let res = fs::remove_dir_all(&endpoint.path());
if let Err(e) = res {
warn!(target: "dapps", "Unable to remove dapp: {:?}", e);
}
}
}
removed.push(entry);
len -= 1;
}
removed
}
#[cfg(test)]
pub fn len(&self) -> usize {
self.cache.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn only_keys(data: Vec<(String, ContentStatus)>) -> Vec<String> {
data.into_iter().map(|x| x.0).collect()
}
#[test]
fn should_remove_least_recently_used() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));
// when
let res = cache.clear_garbage(2);
// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["a"]);
}
#[test]
fn should_update_lru_if_accessed() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));
// when
cache.get("a");
let res = cache.clear_garbage(2);
// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["b"]);
}
}

View File

@@ -23,7 +23,7 @@ use std::{fs, env, fmt};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool};
use rustc_serialize::hex::FromHex;
use hyper::Control;
@@ -33,20 +33,18 @@ use random_filename;
use util::{Mutex, H256};
use util::sha3::sha3;
use page::LocalPageEndpoint;
use handlers::{ContentHandler, AppFetcherHandler, DappHandler};
use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
use endpoint::{Endpoint, EndpointPath, Handler};
use apps::cache::{ContentCache, ContentStatus};
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
use apps::urlhint::{URLHintContract, URLHint};
enum AppStatus {
Fetching,
Ready(LocalPageEndpoint),
}
const MAX_CACHED_DAPPS: usize = 10;
pub struct AppFetcher<R: URLHint = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}
impl<R: URLHint> Drop for AppFetcher<R> {
@@ -65,17 +63,17 @@ impl<R: URLHint> AppFetcher<R> {
AppFetcher {
dapps_path: dapps_path,
resolver: resolver,
dapps: Arc::new(Mutex::new(HashMap::new())),
dapps: Arc::new(Mutex::new(ContentCache::default())),
}
}
#[cfg(test)]
fn set_status(&self, app_id: &str, status: AppStatus) {
fn set_status(&self, app_id: &str, status: ContentStatus) {
self.dapps.lock().insert(app_id.to_owned(), status);
}
pub fn contains(&self, app_id: &str) -> bool {
let dapps = self.dapps.lock();
let mut dapps = self.dapps.lock();
match dapps.get(app_id) {
// Check if we already have the app
Some(_) => true,
@@ -95,11 +93,11 @@ impl<R: URLHint> AppFetcher<R> {
let status = dapps.get(&app_id);
match status {
// Just server dapp
Some(&AppStatus::Ready(ref endpoint)) => {
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_handler(path))
},
// App is already being fetched
Some(&AppStatus::Fetching) => {
Some(&mut ContentStatus::Fetching(_)) => {
(None, Box::new(ContentHandler::html(
StatusCode::ServiceUnavailable,
format!(
@@ -111,11 +109,13 @@ impl<R: URLHint> AppFetcher<R> {
},
// We need to start fetching app
None => {
// TODO [todr] Keep only last N dapps available!
let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let app = self.resolver.resolve(app_hex).expect("to_handler is called only when `contains` returns true.");
(Some(AppStatus::Fetching), Box::new(AppFetcherHandler::new(
let abort = Arc::new(AtomicBool::new(false));
(Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new(
app,
abort,
control,
path.using_dapps_domains,
DappInstaller {
@@ -129,6 +129,7 @@ impl<R: URLHint> AppFetcher<R> {
};
if let Some(status) = new_status {
dapps.clear_garbage(MAX_CACHED_DAPPS);
dapps.insert(app_id, status);
}
@@ -178,7 +179,7 @@ impl From<zip::result::ZipError> for ValidationError {
struct DappInstaller {
dapp_id: String,
dapps_path: PathBuf,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}
impl DappInstaller {
@@ -213,7 +214,7 @@ impl DappInstaller {
}
}
impl DappHandler for DappInstaller {
impl ContentValidator for DappInstaller {
type Error = ValidationError;
fn validate_and_install(&self, app_path: PathBuf) -> Result<Manifest, ValidationError> {
@@ -280,7 +281,7 @@ impl DappHandler for DappInstaller {
Some(manifest) => {
let path = self.dapp_target_path(manifest);
let app = LocalPageEndpoint::new(path, manifest.clone().into());
dapps.insert(self.dapp_id.clone(), AppStatus::Ready(app));
dapps.insert(self.dapp_id.clone(), ContentStatus::Ready(app));
},
// In case of error
None => {
@@ -292,12 +293,13 @@ impl DappHandler for DappInstaller {
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::{AppFetcher, AppStatus};
use apps::urlhint::{GithubApp, URLHint};
use std::env;
use util::Bytes;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use util::Bytes;
use apps::cache::ContentStatus;
use apps::urlhint::{GithubApp, URLHint};
use super::AppFetcher;
struct FakeResolver;
impl URLHint for FakeResolver {
@@ -309,8 +311,9 @@ mod tests {
#[test]
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = AppFetcher::new(FakeResolver);
let handler = LocalPageEndpoint::new(PathBuf::from("/tmp/test"), EndpointInfo {
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),
@@ -319,8 +322,8 @@ mod tests {
});
// when
fetcher.set_status("test", AppStatus::Ready(handler));
fetcher.set_status("test2", AppStatus::Fetching);
fetcher.set_status("test", ContentStatus::Ready(handler));
fetcher.set_status("test2", ContentStatus::Fetching(Default::default()));
// then
assert_eq!(fetcher.contains("test"), true);

View File

@@ -19,6 +19,7 @@ use page::PageEndpoint;
use proxypac::ProxyPac;
use parity_dapps::WebApp;
mod cache;
mod fs;
pub mod urlhint;
pub mod fetcher;

View File

@@ -18,7 +18,8 @@
use std::{env, io, fs, fmt};
use std::path::PathBuf;
use std::sync::mpsc;
use std::sync::{mpsc, Arc};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use random_filename;
@@ -31,6 +32,7 @@ use super::FetchError;
#[derive(Debug)]
pub enum Error {
Aborted,
NotStarted,
UnexpectedStatus(StatusCode),
IoError(io::Error),
@@ -42,6 +44,7 @@ pub type OnDone = Box<Fn() + Send>;
pub struct Fetch {
path: PathBuf,
abort: Arc<AtomicBool>,
file: Option<fs::File>,
result: Option<FetchResult>,
sender: mpsc::Sender<FetchResult>,
@@ -58,7 +61,7 @@ impl Drop for Fetch {
fn drop(&mut self) {
let res = self.result.take().unwrap_or(Err(Error::NotStarted.into()));
// Remove file if there was an error
if res.is_err() {
if res.is_err() || self.is_aborted() {
if let Some(file) = self.file.take() {
drop(file);
// Remove file
@@ -74,12 +77,13 @@ impl Drop for Fetch {
}
impl Fetch {
pub fn new(sender: mpsc::Sender<FetchResult>, on_done: OnDone) -> Self {
pub fn new(sender: mpsc::Sender<FetchResult>, abort: Arc<AtomicBool>, on_done: OnDone) -> Self {
let mut dir = env::temp_dir();
dir.push(random_filename());
Fetch {
path: dir,
abort: abort,
file: None,
result: None,
sender: sender,
@@ -88,17 +92,36 @@ impl Fetch {
}
}
impl Fetch {
fn is_aborted(&self) -> bool {
self.abort.load(Ordering::Relaxed)
}
fn mark_aborted(&mut self) -> Next {
self.result = Some(Err(Error::Aborted));
Next::end()
}
}
impl hyper::client::Handler<HttpStream> for Fetch {
fn on_request(&mut self, req: &mut Request) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
req.headers_mut().set(Connection::close());
read()
}
fn on_request_writable(&mut self, _encoder: &mut Encoder<HttpStream>) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
read()
}
fn on_response(&mut self, res: Response) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
if *res.status() != StatusCode::Ok {
self.result = Some(Err(Error::UnexpectedStatus(*res.status()).into()));
return Next::end();
@@ -119,6 +142,9 @@ impl hyper::client::Handler<HttpStream> for Fetch {
}
fn on_response_readable(&mut self, decoder: &mut Decoder<HttpStream>) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) {
Ok(0) => Next::end(),
Ok(_) => read(),

View File

@@ -18,7 +18,8 @@
use std::fmt;
use std::path::PathBuf;
use std::sync::mpsc;
use std::sync::{mpsc, Arc};
use std::sync::atomic::AtomicBool;
use std::time::{Instant, Duration};
use hyper::{header, server, Decoder, Encoder, Next, Method, Control};
@@ -38,19 +39,20 @@ enum FetchState {
Error(ContentHandler),
InProgress {
deadline: Instant,
receiver: mpsc::Receiver<FetchResult>
receiver: mpsc::Receiver<FetchResult>,
},
Done(Manifest),
}
pub trait DappHandler {
pub trait ContentValidator {
type Error: fmt::Debug + fmt::Display;
fn validate_and_install(&self, app: PathBuf) -> Result<Manifest, Self::Error>;
fn done(&self, Option<&Manifest>);
}
pub struct AppFetcherHandler<H: DappHandler> {
pub struct ContentFetcherHandler<H: ContentValidator> {
abort: Arc<AtomicBool>,
control: Option<Control>,
status: FetchState,
client: Option<Client>,
@@ -58,7 +60,7 @@ pub struct AppFetcherHandler<H: DappHandler> {
dapp: H,
}
impl<H: DappHandler> Drop for AppFetcherHandler<H> {
impl<H: ContentValidator> Drop for ContentFetcherHandler<H> {
fn drop(&mut self) {
let manifest = match self.status {
FetchState::Done(ref manifest) => Some(manifest),
@@ -68,16 +70,18 @@ impl<H: DappHandler> Drop for AppFetcherHandler<H> {
}
}
impl<H: DappHandler> AppFetcherHandler<H> {
impl<H: ContentValidator> ContentFetcherHandler<H> {
pub fn new(
app: GithubApp,
abort: Arc<AtomicBool>,
control: Control,
using_dapps_domains: bool,
handler: H) -> Self {
let client = Client::new();
AppFetcherHandler {
ContentFetcherHandler {
abort: abort,
control: Some(control),
client: Some(client),
status: FetchState::NotStarted(app),
@@ -93,9 +97,8 @@ impl<H: DappHandler> AppFetcherHandler<H> {
}
// TODO [todr] https support
fn fetch_app(client: &mut Client, app: &GithubApp, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
client.request(app.url(), Box::new(move || {
fn fetch_app(client: &mut Client, app: &GithubApp, abort: Arc<AtomicBool>, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
client.request(app.url(), abort, Box::new(move || {
trace!(target: "dapps", "Fetching finished.");
// Ignoring control errors
let _ = control.ready(Next::read());
@@ -103,7 +106,7 @@ impl<H: DappHandler> AppFetcherHandler<H> {
}
}
impl<H: DappHandler> server::Handler<HttpStream> for AppFetcherHandler<H> {
impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<H> {
fn on_request(&mut self, request: server::Request<HttpStream>) -> Next {
let status = if let FetchState::NotStarted(ref app) = self.status {
Some(match *request.method() {
@@ -112,7 +115,7 @@ impl<H: DappHandler> server::Handler<HttpStream> for AppFetcherHandler<H> {
trace!(target: "dapps", "Fetching dapp: {:?}", app);
let control = self.control.take().expect("on_request is called only once, thus control is always Some");
let client = self.client.as_mut().expect("on_request is called before client is closed.");
let fetch = Self::fetch_app(client, app, control);
let fetch = Self::fetch_app(client, app, self.abort.clone(), control);
match fetch {
Ok(receiver) => FetchState::InProgress {
deadline: Instant::now() + Duration::from_secs(FETCH_TIMEOUT),

View File

@@ -27,7 +27,7 @@ pub use self::auth::AuthRequiredHandler;
pub use self::echo::EchoHandler;
pub use self::content::ContentHandler;
pub use self::redirect::Redirection;
pub use self::fetch::{AppFetcherHandler, DappHandler};
pub use self::fetch::{ContentFetcherHandler, ContentValidator};
use url::Url;
use hyper::{server, header, net, uri};

View File

@@ -61,6 +61,7 @@ extern crate parity_dapps;
extern crate https_fetch;
extern crate ethcore_rpc;
extern crate ethcore_util as util;
extern crate linked_hash_map;
mod endpoint;
mod apps;
@@ -109,14 +110,28 @@ impl ServerBuilder {
/// Asynchronously start server with no authentication,
/// returns result with `Server` handle on success or an error.
pub fn start_unsecure_http(&self, addr: &SocketAddr) -> Result<Server, ServerError> {
Server::start_http(addr, NoAuth, self.handler.clone(), self.dapps_path.clone(), self.registrar.clone())
pub fn start_unsecured_http(&self, addr: &SocketAddr, hosts: Option<Vec<String>>) -> Result<Server, ServerError> {
Server::start_http(
addr,
hosts,
NoAuth,
self.handler.clone(),
self.dapps_path.clone(),
self.registrar.clone()
)
}
/// Asynchronously start server with `HTTP Basic Authentication`,
/// return result with `Server` handle on success or an error.
pub fn start_basic_auth_http(&self, addr: &SocketAddr, username: &str, password: &str) -> Result<Server, ServerError> {
Server::start_http(addr, HttpBasicAuth::single_user(username, password), self.handler.clone(), self.dapps_path.clone(), self.registrar.clone())
pub fn start_basic_auth_http(&self, addr: &SocketAddr, hosts: Option<Vec<String>>, username: &str, password: &str) -> Result<Server, ServerError> {
Server::start_http(
addr,
hosts,
HttpBasicAuth::single_user(username, password),
self.handler.clone(),
self.dapps_path.clone(),
self.registrar.clone()
)
}
}
@@ -127,8 +142,24 @@ pub struct Server {
}
impl Server {
/// Returns a list of allowed hosts or `None` if all hosts are allowed.
fn allowed_hosts(hosts: Option<Vec<String>>, bind_address: String) -> Option<Vec<String>> {
let mut allowed = Vec::new();
match hosts {
Some(hosts) => allowed.extend_from_slice(&hosts),
None => return None,
}
// Add localhost domain as valid too if listening on loopback interface.
allowed.push(bind_address.replace("127.0.0.1", "localhost").into());
allowed.push(bind_address.into());
Some(allowed)
}
fn start_http<A: Authorization + 'static>(
addr: &SocketAddr,
hosts: Option<Vec<String>>,
authorization: A,
handler: Arc<IoHandler>,
dapps_path: String,
@@ -145,7 +176,7 @@ impl Server {
special.insert(router::SpecialEndpoint::Utils, apps::utils());
special
});
let bind_address = format!("{}", addr);
let hosts = Self::allowed_hosts(hosts, format!("{}", addr));
try!(hyper::Server::http(addr))
.handle(move |ctrl| router::Router::new(
@@ -155,7 +186,7 @@ impl Server {
endpoints.clone(),
special.clone(),
authorization.clone(),
bind_address.clone(),
hosts.clone(),
))
.map(|(l, srv)| {
@@ -208,3 +239,23 @@ pub fn random_filename() -> String {
rng.gen_ascii_chars().take(12).collect()
}
#[cfg(test)]
mod tests {
use super::Server;
#[test]
fn should_return_allowed_hosts() {
// given
let bind_address = "127.0.0.1".to_owned();
// when
let all = Server::allowed_hosts(None, bind_address.clone());
let address = Server::allowed_hosts(Some(Vec::new()), bind_address.clone());
let some = Server::allowed_hosts(Some(vec!["ethcore.io".into()]), bind_address.clone());
// then
assert_eq!(all, None);
assert_eq!(address, Some(vec!["localhost".into(), "127.0.0.1".into()]));
assert_eq!(some, Some(vec!["ethcore.io".into(), "localhost".into(), "127.0.0.1".into()]));
}
}

View File

@@ -33,6 +33,10 @@ impl LocalPageEndpoint {
info: info,
}
}
pub fn path(&self) -> PathBuf {
self.path.clone()
}
}
impl Endpoint for LocalPageEndpoint {

View File

@@ -22,13 +22,11 @@ use hyper::net::HttpStream;
use jsonrpc_http_server::{is_host_header_valid};
use handlers::ContentHandler;
pub fn is_valid(request: &server::Request<HttpStream>, bind_address: &str, endpoints: Vec<String>) -> bool {
let mut endpoints = endpoints.into_iter()
pub fn is_valid(request: &server::Request<HttpStream>, allowed_hosts: &[String], endpoints: Vec<String>) -> bool {
let mut endpoints = endpoints.iter()
.map(|endpoint| format!("{}{}", endpoint, DAPPS_DOMAIN))
.collect::<Vec<String>>();
// Add localhost domain as valid too if listening on loopback interface.
endpoints.push(bind_address.replace("127.0.0.1", "localhost").into());
endpoints.push(bind_address.into());
endpoints.extend_from_slice(allowed_hosts);
let header_valid = is_host_header_valid(request, &endpoints);

View File

@@ -48,7 +48,7 @@ pub struct Router<A: Authorization + 'static> {
fetch: Arc<AppFetcher>,
special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
authorization: Arc<A>,
bind_address: String,
allowed_hosts: Option<Vec<String>>,
handler: Box<server::Handler<HttpStream> + Send>,
}
@@ -56,9 +56,11 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
fn on_request(&mut self, req: server::Request<HttpStream>) -> Next {
// Validate Host header
if !host_validation::is_valid(&req, &self.bind_address, self.endpoints.keys().cloned().collect()) {
self.handler = host_validation::host_invalid_response();
return self.handler.on_request(req);
if let Some(ref hosts) = self.allowed_hosts {
if !host_validation::is_valid(&req, hosts, self.endpoints.keys().cloned().collect()) {
self.handler = host_validation::host_invalid_response();
return self.handler.on_request(req);
}
}
// Check authorization
@@ -125,7 +127,7 @@ impl<A: Authorization> Router<A> {
endpoints: Arc<Endpoints>,
special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
authorization: Arc<A>,
bind_address: String,
allowed_hosts: Option<Vec<String>>,
) -> Self {
let handler = special.get(&SpecialEndpoint::Rpc).unwrap().to_handler(EndpointPath::default());
@@ -136,7 +138,7 @@ impl<A: Authorization> Router<A> {
fetch: app_fetcher,
special: special,
authorization: authorization,
bind_address: bind_address,
allowed_hosts: allowed_hosts,
handler: handler,
}
}