Update jsonrpc dependencies and rewrite dapps to futures. (#6522)

* Bump version.

* Fix RPC crate.

* Fix BoxFuture in crates.

* Compiles and passes tests!

* Get rid of .boxed()

* Fixing issues with the UI.

* Remove minihttp. Support threads.

* Reimplement files serving to do it in chunks.

* Increase chunk size.

* Remove some unecessary copying.

* Fix tests.

* Fix stratum warning and ipfs todo.

* Switch to proper branch of jsonrpc.

* Update Cargo.lock.

* Update docs.

* Include dapps-glue in workspace.

* fixed merge artifacts

* Fix test compilation.
This commit is contained in:
Tomasz Drwięga
2017-10-05 12:35:01 +02:00
committed by Arkadiy Paronyan
parent 492da38d67
commit e8b418ca03
118 changed files with 2090 additions and 2908 deletions

View File

@@ -19,12 +19,12 @@
use std::fs;
use linked_hash_map::LinkedHashMap;
use page::LocalPageEndpoint;
use page::local;
use handlers::FetchControl;
pub enum ContentStatus {
Fetching(FetchControl),
Ready(LocalPageEndpoint),
Ready(local::Dapp),
}
#[derive(Default)]

View File

@@ -18,16 +18,17 @@ use zip;
use std::{fs, fmt};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use fetch::{self, Mime};
use hash::keccak_buffer;
use bigint::hash::H256;
use fetch::{self, Mime};
use futures_cpupool::CpuPool;
use hash::keccak_buffer;
use page::{LocalPageEndpoint, PageCache};
use handlers::{ContentValidator, ValidatorResponse};
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
use handlers::{ContentValidator, ValidatorResponse};
use page::{local, PageCache};
use Embeddable;
type OnDone = Box<Fn(Option<LocalPageEndpoint>) + Send>;
type OnDone = Box<Fn(Option<local::Dapp>) + Send>;
fn write_response_and_check_hash(
id: &str,
@@ -75,15 +76,17 @@ pub struct Content {
mime: Mime,
content_path: PathBuf,
on_done: OnDone,
pool: CpuPool,
}
impl Content {
pub fn new(id: String, mime: Mime, content_path: PathBuf, on_done: OnDone) -> Self {
pub fn new(id: String, mime: Mime, content_path: PathBuf, on_done: OnDone, pool: CpuPool) -> Self {
Content {
id: id,
mime: mime,
content_path: content_path,
on_done: on_done,
id,
mime,
content_path,
on_done,
pool,
}
}
}
@@ -91,12 +94,15 @@ impl Content {
impl ContentValidator for Content {
type Error = ValidationError;
fn validate_and_install(&self, response: fetch::Response) -> Result<ValidatorResponse, ValidationError> {
let validate = |content_path: PathBuf| {
fn validate_and_install(self, response: fetch::Response) -> Result<ValidatorResponse, ValidationError> {
let pool = self.pool;
let id = self.id.clone();
let mime = self.mime;
let validate = move |content_path: PathBuf| {
// Create dir
let (_, content_path) = write_response_and_check_hash(self.id.as_str(), content_path.clone(), self.id.as_str(), response)?;
let (_, content_path) = write_response_and_check_hash(&id, content_path, &id, response)?;
Ok(LocalPageEndpoint::single_file(content_path, self.mime.clone(), PageCache::Enabled))
Ok(local::Dapp::single_file(pool, content_path, mime, PageCache::Enabled))
};
// Prepare path for a file
@@ -118,15 +124,17 @@ pub struct Dapp {
dapps_path: PathBuf,
on_done: OnDone,
embeddable_on: Embeddable,
pool: CpuPool,
}
impl Dapp {
pub fn new(id: String, dapps_path: PathBuf, on_done: OnDone, embeddable_on: Embeddable) -> Self {
pub fn new(id: String, dapps_path: PathBuf, on_done: OnDone, embeddable_on: Embeddable, pool: CpuPool) -> Self {
Dapp {
id,
dapps_path,
on_done,
embeddable_on,
pool,
}
}
@@ -158,16 +166,19 @@ impl Dapp {
impl ContentValidator for Dapp {
type Error = ValidationError;
fn validate_and_install(&self, response: fetch::Response) -> Result<ValidatorResponse, ValidationError> {
let validate = |dapp_path: PathBuf| {
let (file, zip_path) = write_response_and_check_hash(self.id.as_str(), dapp_path.clone(), &format!("{}.zip", self.id), response)?;
fn validate_and_install(self, response: fetch::Response) -> Result<ValidatorResponse, ValidationError> {
let id = self.id.clone();
let pool = self.pool;
let embeddable_on = self.embeddable_on;
let validate = move |dapp_path: PathBuf| {
let (file, zip_path) = write_response_and_check_hash(&id, dapp_path.clone(), &format!("{}.zip", id), response)?;
trace!(target: "dapps", "Opening dapp bundle at {:?}", zip_path);
// Unpack archive
let mut zip = zip::ZipArchive::new(file)?;
// First find manifest file
let (mut manifest, manifest_dir) = Self::find_manifest(&mut zip)?;
// Overwrite id to match hash
manifest.id = self.id.clone();
manifest.id = id;
// Unpack zip
for i in 0..zip.len() {
@@ -198,7 +209,7 @@ impl ContentValidator for Dapp {
let mut manifest_file = fs::File::create(manifest_path)?;
manifest_file.write_all(manifest_str.as_bytes())?;
// Create endpoint
let endpoint = LocalPageEndpoint::new(dapp_path, manifest.clone().into(), PageCache::Enabled, self.embeddable_on.clone());
let endpoint = local::Dapp::new(pool, dapp_path, manifest.into(), PageCache::Enabled, embeddable_on);
Ok(endpoint)
};

View File

@@ -24,27 +24,25 @@ use std::{fs, env};
use std::path::PathBuf;
use std::sync::Arc;
use rustc_hex::FromHex;
use futures::{future, Future};
use futures_cpupool::CpuPool;
use fetch::{Client as FetchClient, Fetch};
use hash_fetch::urlhint::{URLHintContract, URLHint, URLHintResult};
use parity_reactor::Remote;
use hyper;
use hyper::status::StatusCode;
use hyper::StatusCode;
use {Embeddable, SyncStatus, random_filename};
use parking_lot::Mutex;
use page::LocalPageEndpoint;
use page::local;
use handlers::{ContentHandler, ContentFetcherHandler};
use endpoint::{Endpoint, EndpointPath, Handler};
use endpoint::{self, Endpoint, EndpointPath};
use apps::cache::{ContentCache, ContentStatus};
/// Limit of cached dapps/content
const MAX_CACHED_DAPPS: usize = 20;
pub trait Fetcher: Send + Sync + 'static {
pub trait Fetcher: Endpoint + 'static {
fn contains(&self, content_id: &str) -> bool;
fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler>;
}
pub struct ContentFetcher<F: Fetch = FetchClient, R: URLHint + 'static = URLHintContract> {
@@ -53,8 +51,8 @@ pub struct ContentFetcher<F: Fetch = FetchClient, R: URLHint + 'static = URLHint
cache: Arc<Mutex<ContentCache>>,
sync: Arc<SyncStatus>,
embeddable_on: Embeddable,
remote: Remote,
fetch: F,
pool: CpuPool,
only_content: bool,
}
@@ -66,24 +64,23 @@ impl<R: URLHint + 'static, F: Fetch> Drop for ContentFetcher<F, R> {
}
impl<R: URLHint + 'static, F: Fetch> ContentFetcher<F, R> {
pub fn new(
resolver: R,
sync_status: Arc<SyncStatus>,
remote: Remote,
sync: Arc<SyncStatus>,
fetch: F,
pool: CpuPool,
) -> Self {
let mut cache_path = env::temp_dir();
cache_path.push(random_filename());
ContentFetcher {
cache_path: cache_path,
resolver: resolver,
sync: sync_status,
cache_path,
resolver,
sync,
cache: Arc::new(Mutex::new(ContentCache::default())),
embeddable_on: None,
remote: remote,
fetch: fetch,
fetch,
pool,
only_content: true,
}
}
@@ -98,24 +95,34 @@ impl<R: URLHint + 'static, F: Fetch> ContentFetcher<F, R> {
self
}
fn still_syncing(embeddable: Embeddable) -> Box<Handler> {
Box::new(ContentHandler::error(
fn not_found(embeddable: Embeddable) -> endpoint::Response {
Box::new(future::ok(ContentHandler::error(
StatusCode::NotFound,
"Resource Not Found",
"Requested resource was not found.",
None,
embeddable,
).into()))
}
fn still_syncing(embeddable: Embeddable) -> endpoint::Response {
Box::new(future::ok(ContentHandler::error(
StatusCode::ServiceUnavailable,
"Sync In Progress",
"Your node is still syncing. We cannot resolve any content before it's fully synced.",
Some("<a href=\"javascript:window.location.reload()\">Refresh</a>"),
embeddable,
))
).into()))
}
fn dapps_disabled(address: Embeddable) -> Box<Handler> {
Box::new(ContentHandler::error(
fn dapps_disabled(address: Embeddable) -> endpoint::Response {
Box::new(future::ok(ContentHandler::error(
StatusCode::ServiceUnavailable,
"Network Dapps Not Available",
"This interface doesn't support network dapps for security reasons.",
None,
address,
))
).into()))
}
#[cfg(test)]
@@ -126,8 +133,6 @@ impl<R: URLHint + 'static, F: Fetch> ContentFetcher<F, R> {
// resolve contract call synchronously.
// TODO: port to futures-based hyper and make it all async.
fn resolve(&self, content_id: Vec<u8>) -> Option<URLHintResult> {
use futures::Future;
self.resolver.resolve(content_id)
.wait()
.unwrap_or_else(|e| { warn!("Error resolving content-id: {}", e); None })
@@ -151,8 +156,10 @@ impl<R: URLHint + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
false
}
}
}
fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
impl<R: URLHint + 'static, F: Fetch> Endpoint for ContentFetcher<F, R> {
fn respond(&self, path: EndpointPath, req: endpoint::Request) -> endpoint::Response {
let mut cache = self.cache.lock();
let content_id = path.app_id.clone();
@@ -161,12 +168,12 @@ impl<R: URLHint + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
match status {
// Just serve the content
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_async_handler(path, control))
(None, endpoint.to_response(&path))
},
// Content is already being fetched
Some(&mut ContentStatus::Fetching(ref fetch_control)) if !fetch_control.is_deadline_reached() => {
trace!(target: "dapps", "Content fetching in progress. Waiting...");
(None, fetch_control.to_async_handler(path, control))
(None, fetch_control.to_response(path))
},
// We need to start fetching the content
_ => {
@@ -176,7 +183,7 @@ impl<R: URLHint + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
let cache = self.cache.clone();
let id = content_id.clone();
let on_done = move |result: Option<LocalPageEndpoint>| {
let on_done = move |result: Option<local::Dapp>| {
let mut cache = cache.lock();
match result {
Some(endpoint) => cache.insert(id.clone(), ContentStatus::Ready(endpoint)),
@@ -195,39 +202,39 @@ impl<R: URLHint + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
},
Some(URLHintResult::Dapp(dapp)) => {
let handler = ContentFetcherHandler::new(
dapp.url(),
req.method(),
&dapp.url(),
path,
control,
installers::Dapp::new(
content_id.clone(),
self.cache_path.clone(),
Box::new(on_done),
self.embeddable_on.clone(),
self.pool.clone(),
),
self.embeddable_on.clone(),
self.remote.clone(),
self.fetch.clone(),
);
(Some(ContentStatus::Fetching(handler.fetch_control())), Box::new(handler) as Box<Handler>)
(Some(ContentStatus::Fetching(handler.fetch_control())), Box::new(handler) as endpoint::Response)
},
Some(URLHintResult::Content(content)) => {
let handler = ContentFetcherHandler::new(
content.url,
req.method(),
&content.url,
path,
control,
installers::Content::new(
content_id.clone(),
content.mime,
self.cache_path.clone(),
Box::new(on_done),
self.pool.clone(),
),
self.embeddable_on.clone(),
self.remote.clone(),
self.fetch.clone(),
);
(Some(ContentStatus::Fetching(handler.fetch_control())), Box::new(handler) as Box<Handler>)
(Some(ContentStatus::Fetching(handler.fetch_control())), Box::new(handler) as endpoint::Response)
},
None if self.sync.is_major_importing() => {
(None, Self::still_syncing(self.embeddable_on.clone()))
@@ -235,13 +242,7 @@ impl<R: URLHint + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
None => {
// This may happen when sync status changes in between
// `contains` and `to_handler`
(None, Box::new(ContentHandler::error(
StatusCode::NotFound,
"Resource Not Found",
"Requested resource was not found.",
None,
self.embeddable_on.clone(),
)) as Box<Handler>)
(None, Self::not_found(self.embeddable_on.clone()))
},
}
},
@@ -263,13 +264,12 @@ mod tests {
use std::sync::Arc;
use bytes::Bytes;
use fetch::{Fetch, Client};
use futures::{future, Future, BoxFuture};
use hash_fetch::urlhint::{URLHint, URLHintResult};
use parity_reactor::Remote;
use futures::future;
use hash_fetch::urlhint::{URLHint, URLHintResult, BoxFuture};
use apps::cache::ContentStatus;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use page::local;
use super::{ContentFetcher, Fetcher};
use {SyncStatus};
@@ -277,7 +277,7 @@ mod tests {
struct FakeResolver;
impl URLHint for FakeResolver {
fn resolve(&self, _id: Bytes) -> BoxFuture<Option<URLHintResult>, String> {
future::ok(None).boxed()
Box::new(future::ok(None))
}
}
@@ -291,10 +291,16 @@ mod tests {
#[test]
fn should_true_if_contains_the_app() {
// given
let pool = ::futures_cpupool::CpuPool::new(1);
let path = env::temp_dir();
let fetcher = ContentFetcher::new(FakeResolver, Arc::new(FakeSync(false)), Remote::new_sync(), Client::new().unwrap())
.allow_dapps(true);
let handler = LocalPageEndpoint::new(path, EndpointInfo {
let fetcher = ContentFetcher::new(
FakeResolver,
Arc::new(FakeSync(false)),
Client::new().unwrap(),
pool.clone(),
).allow_dapps(true);
let handler = local::Dapp::new(pool, path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),

View File

@@ -19,9 +19,11 @@ use std::io;
use std::io::Read;
use std::fs;
use std::path::{Path, PathBuf};
use page::{LocalPageEndpoint, PageCache};
use endpoint::{Endpoint, EndpointInfo};
use futures_cpupool::CpuPool;
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest};
use endpoint::{Endpoint, EndpointInfo};
use page::{local, PageCache};
use Embeddable;
struct LocalDapp {
@@ -61,14 +63,14 @@ fn read_manifest(name: &str, mut path: PathBuf) -> EndpointInfo {
/// Returns Dapp Id and Local Dapp Endpoint for given filesystem path.
/// Parses the path to extract last component (for name).
/// `None` is returned when path is invalid or non-existent.
pub fn local_endpoint<P: AsRef<Path>>(path: P, embeddable: Embeddable) -> Option<(String, Box<LocalPageEndpoint>)> {
pub fn local_endpoint<P: AsRef<Path>>(path: P, embeddable: Embeddable, pool: CpuPool) -> Option<(String, Box<local::Dapp>)> {
let path = path.as_ref().to_owned();
path.canonicalize().ok().and_then(|path| {
let name = path.file_name().and_then(|name| name.to_str());
name.map(|name| {
let dapp = local_dapp(name.into(), path.clone());
(dapp.id, Box::new(LocalPageEndpoint::new(
dapp.path, dapp.info, PageCache::Disabled, embeddable.clone())
(dapp.id, Box::new(local::Dapp::new(
pool.clone(), dapp.path, dapp.info, PageCache::Disabled, embeddable.clone())
))
})
})
@@ -86,13 +88,13 @@ fn local_dapp(name: String, path: PathBuf) -> LocalDapp {
}
/// Returns endpoints for Local Dapps found for given filesystem path.
/// Scans the directory and collects `LocalPageEndpoints`.
pub fn local_endpoints<P: AsRef<Path>>(dapps_path: P, embeddable: Embeddable) -> BTreeMap<String, Box<Endpoint>> {
/// Scans the directory and collects `local::Dapp`.
pub fn local_endpoints<P: AsRef<Path>>(dapps_path: P, embeddable: Embeddable, pool: CpuPool) -> BTreeMap<String, Box<Endpoint>> {
let mut pages = BTreeMap::<String, Box<Endpoint>>::new();
for dapp in local_dapps(dapps_path.as_ref()) {
pages.insert(
dapp.id,
Box::new(LocalPageEndpoint::new(dapp.path, dapp.info, PageCache::Disabled, embeddable.clone()))
Box::new(local::Dapp::new(pool.clone(), dapp.path, dapp.info, PageCache::Disabled, embeddable.clone()))
);
}
pages

View File

@@ -18,12 +18,12 @@ use std::path::PathBuf;
use std::sync::Arc;
use endpoint::{Endpoints, Endpoint};
use page::PageEndpoint;
use futures_cpupool::CpuPool;
use page;
use proxypac::ProxyPac;
use web::Web;
use fetch::Fetch;
use parity_dapps::WebApp;
use parity_reactor::Remote;
use parity_ui;
use {WebProxyTokens, ParentFrameSettings};
@@ -43,12 +43,12 @@ pub const UTILS_PATH: &'static str = "parity-utils";
pub const WEB_PATH: &'static str = "web";
pub const URL_REFERER: &'static str = "__referer=";
pub fn utils() -> Box<Endpoint> {
Box::new(PageEndpoint::with_prefix(parity_ui::App::default(), UTILS_PATH.to_owned()))
pub fn utils(pool: CpuPool) -> Box<Endpoint> {
Box::new(page::builtin::Dapp::new(pool, parity_ui::App::default()))
}
pub fn ui() -> Box<Endpoint> {
Box::new(PageEndpoint::with_fallback_to_index(parity_ui::App::default()))
pub fn ui(pool: CpuPool) -> Box<Endpoint> {
Box::new(page::builtin::Dapp::with_fallback_to_index(pool, parity_ui::App::default()))
}
pub fn ui_redirection(embeddable: Option<ParentFrameSettings>) -> Box<Endpoint> {
@@ -61,14 +61,14 @@ pub fn all_endpoints<F: Fetch>(
dapps_domain: &str,
embeddable: Option<ParentFrameSettings>,
web_proxy_tokens: Arc<WebProxyTokens>,
remote: Remote,
fetch: F,
pool: CpuPool,
) -> (Vec<String>, Endpoints) {
// fetch fs dapps at first to avoid overwriting builtins
let mut pages = fs::local_endpoints(dapps_path.clone(), embeddable.clone());
let mut pages = fs::local_endpoints(dapps_path.clone(), embeddable.clone(), pool.clone());
let local_endpoints: Vec<String> = pages.keys().cloned().collect();
for path in extra_dapps {
if let Some((id, endpoint)) = fs::local_endpoint(path.clone(), embeddable.clone()) {
if let Some((id, endpoint)) = fs::local_endpoint(path.clone(), embeddable.clone(), pool.clone()) {
pages.insert(id, endpoint);
} else {
warn!(target: "dapps", "Ignoring invalid dapp at {}", path.display());
@@ -76,17 +76,17 @@ pub fn all_endpoints<F: Fetch>(
}
// NOTE [ToDr] Dapps will be currently embeded on 8180
insert::<parity_ui::App>(&mut pages, "ui", Embeddable::Yes(embeddable.clone()));
insert::<parity_ui::App>(&mut pages, "ui", Embeddable::Yes(embeddable.clone()), pool.clone());
pages.insert("proxy".into(), ProxyPac::boxed(embeddable.clone(), dapps_domain.to_owned()));
pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), remote.clone(), fetch.clone()));
pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), fetch.clone()));
(local_endpoints, pages)
}
fn insert<T : WebApp + Default + 'static>(pages: &mut Endpoints, id: &str, embed_at: Embeddable) {
fn insert<T : WebApp + Default + 'static>(pages: &mut Endpoints, id: &str, embed_at: Embeddable, pool: CpuPool) {
pages.insert(id.to_owned(), Box::new(match embed_at {
Embeddable::Yes(address) => PageEndpoint::new_safe_to_embed(T::default(), address),
Embeddable::No => PageEndpoint::new(T::default()),
Embeddable::Yes(address) => page::builtin::Dapp::new_safe_to_embed(pool, T::default(), address),
Embeddable::No => page::builtin::Dapp::new(pool, T::default()),
}));
}

View File

@@ -16,9 +16,10 @@
//! UI redirections
use hyper::{Control, StatusCode};
use hyper::StatusCode;
use futures::future;
use endpoint::{Endpoint, Handler, EndpointPath};
use endpoint::{Endpoint, Request, Response, EndpointPath};
use {handlers, Embeddable};
/// Redirection to UI server.
@@ -37,19 +38,20 @@ impl Redirection {
}
impl Endpoint for Redirection {
fn to_async_handler(&self, _path: EndpointPath, _control: Control) -> Box<Handler> {
if let Some(ref frame) = self.embeddable_on {
fn respond(&self, _path: EndpointPath, req: Request) -> Response {
Box::new(future::ok(if let Some(ref frame) = self.embeddable_on {
trace!(target: "dapps", "Redirecting to signer interface.");
handlers::Redirection::boxed(&format!("http://{}:{}", &frame.host, frame.port))
let protocol = req.uri().scheme().unwrap_or("http");
handlers::Redirection::new(format!("{}://{}:{}", protocol, &frame.host, frame.port)).into()
} else {
trace!(target: "dapps", "Signer disabled, returning 404.");
Box::new(handlers::ContentHandler::error(
handlers::ContentHandler::error(
StatusCode::NotFound,
"404 Not Found",
"Your homepage is not available when Trusted Signer is disabled.",
Some("You can still access dapps by writing a correct address, though. Re-enable Signer to get your homepage back."),
None,
))
}
).into()
}))
}
}