From 0620a03e56f1db3515b9371144e0b0c666ac5aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 18 Aug 2016 12:19:09 +0200 Subject: [PATCH] Fetchable dapps (#1949) * Fetching dapp from github. * Unpacking dapp * Removing hardcodes * Proper Host validation * Randomizing paths * Splitting into files * Serving donwloaded apps from different path * Extracting URLHint to separate module * Whitespace and docs --- Cargo.lock | 51 ++++ dapps/Cargo.toml | 2 + dapps/src/api/types.rs.in | 14 +- dapps/src/apps/fetcher.rs | 298 ++++++++++++++++++++++++ dapps/src/apps/fs.rs | 15 +- dapps/src/apps/manifest.rs | 29 +++ dapps/src/apps/mod.rs | 3 + dapps/src/apps/urlhint.rs | 104 +++++++++ dapps/src/endpoint.rs | 8 +- dapps/src/handlers/client/fetch_file.rs | 141 +++++++++++ dapps/src/handlers/client/mod.rs | 22 ++ dapps/src/handlers/content.rs | 4 + dapps/src/handlers/fetch.rs | 226 ++++++++++++++++++ dapps/src/handlers/mod.rs | 3 + dapps/src/lib.rs | 18 +- dapps/src/router/host_validation.rs | 11 +- dapps/src/router/mod.rs | 14 +- 17 files changed, 941 insertions(+), 22 deletions(-) create mode 100644 dapps/src/apps/fetcher.rs create mode 100644 dapps/src/apps/manifest.rs create mode 100644 dapps/src/apps/urlhint.rs create mode 100644 dapps/src/handlers/client/fetch_file.rs create mode 100644 dapps/src/handlers/client/mod.rs create mode 100644 dapps/src/handlers/fetch.rs diff --git a/Cargo.lock b/Cargo.lock index d919ed94b..82061f368 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,7 @@ dependencies = [ "parity-dapps-home 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", "parity-dapps-status 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", "parity-dapps-wallet 1.4.0 (git+https://github.com/ethcore/parity-ui.git)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde_codegen 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -293,6 +294,7 @@ dependencies = [ "syntex 0.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zip 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -549,6 +551,15 @@ dependencies = [ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "flate2" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "gcc" version = "0.3.28" @@ -780,6 +791,15 @@ dependencies = [ "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mio" version = "0.5.1" @@ -838,6 +858,16 @@ dependencies = [ "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "msdos_time" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nanomsg" version = "0.5.1" @@ -1081,6 +1111,11 @@ dependencies = [ "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "podio" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "primal" version = "0.2.3" @@ -1587,6 +1622,17 @@ dependencies = [ "xml-rs 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "zip" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", + "msdos_time 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67077478f0a03952bed2e6786338d400d40c25e9836e08ad50af96607317fd03" "checksum ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f46cd5b1d660c938e3f92dfe7a73d832b3281479363dd0cd9c1c2fbf60f7962" @@ -1612,6 +1658,7 @@ dependencies = [ "checksum elastic-array 0.4.0 (git+https://github.com/ethcore/elastic-array)" = "" "checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5" "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "" +"checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb" "checksum gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)" = "3da3a2cbaeb01363c8e3704fd9fd0eb2ceb17c6f27abd4c1ef040fb57d20dc79" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" @@ -1637,10 +1684,12 @@ dependencies = [ "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a74cc2587bf97c49f3f5bab62860d6abf3902ca73b66b51d9b049fbdcd727bd2" "checksum mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e50bf542f81754ef69e5cea856946a3819f7c09ea97b4903c8bc8a89f74e7b6" +"checksum miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d1f4d337a01c32e1f2122510fed46393d53ca35a7f429cb0450abaedfa3ed54" "checksum mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)" = "" "checksum mio 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a637d1ca14eacae06296a008fa7ad955347e34efcb5891cfd8ba05491a37907e" "checksum mio 0.6.0-dev (git+https://github.com/carllerche/mio?rev=62ec763c9cc34d8a452ed0392c575c50ddd5fc8d)" = "" "checksum miow 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d5bfc6782530ac8ace97af10a540054a37126b63b0702ddaaa243b73b5745b9a" +"checksum msdos_time 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c04b68cc63a8480fb2550343695f7be72effdec953a9d4508161c3e69041c7d8" "checksum nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)" = "" "checksum nanomsg-sys 0.5.0 (git+https://github.com/ethcore/nanomsg.rs.git)" = "" "checksum net2 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "6a816012ca11cb47009693c1e0c6130e26d39e4d97ee2a13c50e868ec83e3204" @@ -1668,6 +1717,7 @@ dependencies = [ "checksum phf_codegen 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8af7ae7c3f75a502292b491e5cc0a1f69e3407744abe6e57e2a3b712bb82f01d" "checksum phf_generator 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "db005608fd99800c8c74106a7c894cf582055b689aa14a79462cefdcb7dc1cc3" "checksum phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "fee4d039930e4f45123c9b15976cf93a499847b6483dc09c42ea0ec4940f2aa6" +"checksum podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e5422a1ee1bc57cc47ae717b0137314258138f38fd5f3cea083f43a9725383a0" "checksum primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0e31b86efadeaeb1235452171a66689682783149a6249ff334a2c5d8218d00a4" "checksum primal-bit 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "464a91febc06166783d4f5ba3577b5ed8dda8e421012df80bfe48a971ed7be8f" "checksum primal-check 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "647c81b67bb9551a7b88d0bcd785ac35b7d0bf4b2f358683d7c2375d04daec51" @@ -1732,3 +1782,4 @@ dependencies = [ "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum xml-rs 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "65e74b96bd3179209dc70a980da6df843dff09e46eee103a0376c0949257e3ef" "checksum xmltree 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "472a9d37c7c53ab2391161df5b89b1f3bf76dab6ab150d7941ecbdd832282082" +"checksum zip 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3ceb33a75b3d0608942302eed325b59d2c3ed777cc6c01627ae14e5697c6a31c" diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 13b524f73..3bd5d0875 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -9,6 +9,7 @@ build = "build.rs" [lib] [dependencies] +rand = "0.3.14" log = "0.3" jsonrpc-core = "2.1" jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc-http-server.git" } @@ -19,6 +20,7 @@ rustc-serialize = "0.3" serde = "0.7.0" serde_json = "0.7.0" serde_macros = { version = "0.7.0", optional = true } +zip = { version = "0.1", default-features = false } ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } diff --git a/dapps/src/api/types.rs.in b/dapps/src/api/types.rs.in index a7961a144..8bbefaa83 100644 --- a/dapps/src/api/types.rs.in +++ b/dapps/src/api/types.rs.in @@ -16,7 +16,7 @@ use endpoint::EndpointInfo; -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub struct App { pub id: String, pub name: String, @@ -41,6 +41,18 @@ impl App { } } +impl Into for App { + fn into(self) -> EndpointInfo { + EndpointInfo { + name: self.name, + description: self.description, + version: self.version, + author: self.author, + icon_url: self.icon_url, + } + } +} + #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct ApiError { pub code: String, diff --git a/dapps/src/apps/fetcher.rs b/dapps/src/apps/fetcher.rs new file mode 100644 index 000000000..2b1b9e658 --- /dev/null +++ b/dapps/src/apps/fetcher.rs @@ -0,0 +1,298 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Fetchable Dapps support. +//! Manages downloaded (cached) Dapps and downloads them when necessary. +//! Uses `URLHint` to resolve addresses into Dapps bundle file location. + +use zip; +use std::{fs, env}; +use std::io::{self, Read, Write}; +use std::path::PathBuf; +use std::sync::Arc; +use std::collections::HashMap; + +use hyper::Control; +use hyper::status::StatusCode; + +use random_filename; +use util::Mutex; +use page::LocalPageEndpoint; +use handlers::{ContentHandler, AppFetcherHandler, DappHandler}; +use endpoint::{Endpoint, EndpointPath, Handler}; +use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest}; +use apps::urlhint::{URLHintContract, URLHint}; + +enum AppStatus { + Fetching, + Ready(LocalPageEndpoint), +} + +pub struct AppFetcher { + dapps_path: PathBuf, + resolver: R, + dapps: Arc>>, +} + +impl Drop for AppFetcher { + fn drop(&mut self) { + // Clear cache path + let _ = fs::remove_dir_all(&self.dapps_path); + } +} + +impl Default for AppFetcher { + fn default() -> Self { + AppFetcher::new(URLHintContract) + } +} + +impl AppFetcher { + + pub fn new(resolver: R) -> Self { + let mut dapps_path = env::temp_dir(); + dapps_path.push(random_filename()); + + AppFetcher { + dapps_path: dapps_path, + resolver: resolver, + dapps: Arc::new(Mutex::new(HashMap::new())), + } + } + + #[cfg(test)] + fn set_status(&self, app_id: &str, status: AppStatus) { + self.dapps.lock().insert(app_id.to_owned(), status); + } + + pub fn contains(&self, app_id: &str) -> bool { + let dapps = self.dapps.lock(); + match dapps.get(app_id) { + // Check if we already have the app + Some(_) => true, + // fallback to resolver + None => self.resolver.resolve(app_id).is_some(), + } + } + + pub fn to_handler(&self, path: EndpointPath, control: Control) -> Box { + let mut dapps = self.dapps.lock(); + let app_id = path.app_id.clone(); + + let (new_status, handler) = { + let status = dapps.get(&app_id); + match status { + // Just server dapp + Some(&AppStatus::Ready(ref endpoint)) => { + (None, endpoint.to_handler(path)) + }, + // App is already being fetched + Some(&AppStatus::Fetching) => { + (None, Box::new(ContentHandler::html( + StatusCode::ServiceUnavailable, + "

This dapp is already being downloaded.

".into() + )) as Box) + }, + // We need to start fetching app + None => { + // TODO [todr] Keep only last N dapps available! + let app = self.resolver.resolve(&app_id).expect("to_handler is called only when `contains` returns true."); + (Some(AppStatus::Fetching), Box::new(AppFetcherHandler::new( + app, + control, + DappInstaller { + dapp_id: app_id.clone(), + dapps_path: self.dapps_path.clone(), + dapps: self.dapps.clone(), + } + )) as Box) + }, + } + }; + + if let Some(status) = new_status { + dapps.insert(app_id, status); + } + + handler + } +} + +#[derive(Debug)] +pub enum ValidationError { + ManifestNotFound, + ManifestSerialization(String), + Io(io::Error), + Zip(zip::result::ZipError), +} + +impl From for ValidationError { + fn from(err: io::Error) -> Self { + ValidationError::Io(err) + } +} + +impl From for ValidationError { + fn from(err: zip::result::ZipError) -> Self { + ValidationError::Zip(err) + } +} + +struct DappInstaller { + dapp_id: String, + dapps_path: PathBuf, + dapps: Arc>>, +} + +impl DappInstaller { + fn find_manifest(zip: &mut zip::ZipArchive) -> Result<(Manifest, PathBuf), ValidationError> { + for i in 0..zip.len() { + let mut file = try!(zip.by_index(i)); + + if !file.name().ends_with(MANIFEST_FILENAME) { + continue; + } + + // try to read manifest + let mut manifest = String::new(); + let manifest = file + .read_to_string(&mut manifest).ok() + .and_then(|_| deserialize_manifest(manifest).ok()); + + if let Some(manifest) = manifest { + let mut manifest_location = PathBuf::from(file.name()); + manifest_location.pop(); // get rid of filename + return Ok((manifest, manifest_location)); + } + } + + Err(ValidationError::ManifestNotFound) + } + + fn dapp_target_path(&self, manifest: &Manifest) -> PathBuf { + let mut target = self.dapps_path.clone(); + target.push(&manifest.id); + target + } +} + +impl DappHandler for DappInstaller { + type Error = ValidationError; + + fn validate_and_install(&self, app_path: PathBuf) -> Result { + trace!(target: "dapps", "Opening dapp bundle at {:?}", app_path); + // TODO [ToDr] Validate file hash + let file = try!(fs::File::open(app_path)); + // Unpack archive + let mut zip = try!(zip::ZipArchive::new(file)); + // First find manifest file + let (mut manifest, manifest_dir) = try!(Self::find_manifest(&mut zip)); + // Overwrite id to match hash + manifest.id = self.dapp_id.clone(); + + let target = self.dapp_target_path(&manifest); + + // Remove old directory + if target.exists() { + warn!(target: "dapps", "Overwriting existing dapp: {}", manifest.id); + try!(fs::remove_dir_all(target.clone())); + } + + // Unpack zip + for i in 0..zip.len() { + let mut file = try!(zip.by_index(i)); + // TODO [todr] Check if it's consistent on windows. + let is_dir = file.name().chars().rev().next() == Some('/'); + + let file_path = PathBuf::from(file.name()); + let location_in_manifest_base = file_path.strip_prefix(&manifest_dir); + // Create files that are inside manifest directory + if let Ok(location_in_manifest_base) = location_in_manifest_base { + let p = target.join(location_in_manifest_base); + // Check if it's a directory + if is_dir { + try!(fs::create_dir_all(p)); + } else { + let mut target = try!(fs::File::create(p)); + try!(io::copy(&mut file, &mut target)); + } + } + } + + // Write manifest + let manifest_str = try!(serialize_manifest(&manifest).map_err(ValidationError::ManifestSerialization)); + let manifest_path = target.join(MANIFEST_FILENAME); + let mut manifest_file = try!(fs::File::create(manifest_path)); + try!(manifest_file.write_all(manifest_str.as_bytes())); + + // Return modified app manifest + Ok(manifest) + } + + fn done(&self, manifest: Option<&Manifest>) { + let mut dapps = self.dapps.lock(); + match manifest { + Some(manifest) => { + let path = self.dapp_target_path(manifest); + let app = LocalPageEndpoint::new(path, manifest.clone().into()); + dapps.insert(self.dapp_id.clone(), AppStatus::Ready(app)); + }, + // In case of error + None => { + dapps.remove(&self.dapp_id); + }, + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use super::{AppFetcher, AppStatus}; + use apps::urlhint::{GithubApp, URLHint}; + use endpoint::EndpointInfo; + use page::LocalPageEndpoint; + + struct FakeResolver; + impl URLHint for FakeResolver { + fn resolve(&self, _app_id: &str) -> Option { + None + } + } + + #[test] + fn should_true_if_contains_the_app() { + // given + let fetcher = AppFetcher::new(FakeResolver); + let handler = LocalPageEndpoint::new(PathBuf::from("/tmp/test"), EndpointInfo { + name: "fake".into(), + description: "".into(), + version: "".into(), + author: "".into(), + icon_url: "".into(), + }); + + // when + fetcher.set_status("test", AppStatus::Ready(handler)); + fetcher.set_status("test2", AppStatus::Fetching); + + // then + assert_eq!(fetcher.contains("test"), true); + assert_eq!(fetcher.contains("test2"), true); + assert_eq!(fetcher.contains("test3"), false); + } +} + diff --git a/dapps/src/apps/fs.rs b/dapps/src/apps/fs.rs index fa3b0ab4c..4728757de 100644 --- a/dapps/src/apps/fs.rs +++ b/dapps/src/apps/fs.rs @@ -14,14 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use serde_json; use std::io; use std::io::Read; use std::fs; use std::path::PathBuf; use page::LocalPageEndpoint; use endpoint::{Endpoints, EndpointInfo}; -use api::App; +use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest}; struct LocalDapp { id: String, @@ -73,7 +72,7 @@ fn local_dapps(dapps_path: String) -> Vec { } fn read_manifest(name: &str, mut path: PathBuf) -> EndpointInfo { - path.push("manifest.json"); + path.push(MANIFEST_FILENAME); fs::File::open(path.clone()) .map_err(|e| format!("{:?}", e)) @@ -82,15 +81,9 @@ fn read_manifest(name: &str, mut path: PathBuf) -> EndpointInfo { let mut s = String::new(); try!(f.read_to_string(&mut s).map_err(|e| format!("{:?}", e))); // Try to deserialize manifest - serde_json::from_str::(&s).map_err(|e| format!("{:?}", e)) - }) - .map(|app| EndpointInfo { - name: app.name, - description: app.description, - version: app.version, - author: app.author, - icon_url: app.icon_url, + deserialize_manifest(s) }) + .map(Into::into) .unwrap_or_else(|e| { warn!(target: "dapps", "Cannot read manifest file at: {:?}. Error: {:?}", path, e); diff --git a/dapps/src/apps/manifest.rs b/dapps/src/apps/manifest.rs new file mode 100644 index 000000000..6d9874b86 --- /dev/null +++ b/dapps/src/apps/manifest.rs @@ -0,0 +1,29 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use serde_json; +pub use api::App as Manifest; + +pub const MANIFEST_FILENAME: &'static str = "manifest.json"; + +pub fn deserialize_manifest(manifest: String) -> Result { + serde_json::from_str::(&manifest).map_err(|e| format!("{:?}", e)) + // TODO [todr] Manifest validation (especialy: id (used as path)) +} + +pub fn serialize_manifest(manifest: &Manifest) -> Result { + serde_json::to_string_pretty(manifest).map_err(|e| format!("{:?}", e)) +} diff --git a/dapps/src/apps/mod.rs b/dapps/src/apps/mod.rs index 50bdd2073..97b018e68 100644 --- a/dapps/src/apps/mod.rs +++ b/dapps/src/apps/mod.rs @@ -20,6 +20,9 @@ use proxypac::ProxyPac; use parity_dapps::WebApp; mod fs; +pub mod urlhint; +pub mod fetcher; +pub mod manifest; extern crate parity_dapps_status; extern crate parity_dapps_home; diff --git a/dapps/src/apps/urlhint.rs b/dapps/src/apps/urlhint.rs new file mode 100644 index 000000000..61dbc0dec --- /dev/null +++ b/dapps/src/apps/urlhint.rs @@ -0,0 +1,104 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use rustc_serialize::hex::ToHex; + +use util::{Address, FromHex}; + +const COMMIT_LEN: usize = 20; + +#[derive(Debug)] +pub struct GithubApp { + pub account: String, + pub repo: String, + pub commit: [u8;COMMIT_LEN], + pub owner: Address, +} + +impl GithubApp { + pub fn url(&self) -> String { + // format!("https://github.com/{}/{}/archive/{}.zip", self.account, self.repo, self.commit.to_hex()) + format!("http://github.todr.me/{}/{}/zip/{}", self.account, self.repo, self.commit.to_hex()) + } + + fn commit(bytes: &[u8]) -> Option<[u8;COMMIT_LEN]> { + if bytes.len() < COMMIT_LEN { + return None; + } + + let mut commit = [0; COMMIT_LEN]; + for i in 0..COMMIT_LEN { + commit[i] = bytes[i]; + } + + Some(commit) + } +} + +pub trait URLHint { + fn resolve(&self, app_id: &str) -> Option; +} + +pub struct URLHintContract; + +impl URLHint for URLHintContract { + fn resolve(&self, app_id: &str) -> Option { + // TODO [todr] use GithubHint contract to check the details + // For now we are just accepting patterns: ...parity + let mut app_parts = app_id.split('.'); + + let hash = app_parts.next() + .and_then(|h| h.from_hex().ok()) + .and_then(|h| GithubApp::commit(&h)); + let repo = app_parts.next(); + let account = app_parts.next(); + + match (hash, repo, account) { + (Some(hash), Some(repo), Some(account)) => { + Some(GithubApp { + account: account.into(), + repo: repo.into(), + commit: hash, + owner: Address::default(), + }) + }, + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::GithubApp; + use util::Address; + + #[test] + fn should_return_valid_url() { + // given + let app = GithubApp { + account: "test".into(), + repo: "xyz".into(), + commit: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + owner: Address::default(), + }; + + // when + let url = app.url(); + + // then + assert_eq!(url, "http://github.todr.me/test/xyz/zip/000102030405060708090a0b0c0d0e0f10111213".to_owned()); + } +} diff --git a/dapps/src/endpoint.rs b/dapps/src/endpoint.rs index ba2116e1c..62816b088 100644 --- a/dapps/src/endpoint.rs +++ b/dapps/src/endpoint.rs @@ -35,11 +35,11 @@ pub struct EndpointInfo { pub icon_url: String, } +pub type Endpoints = BTreeMap>; +pub type Handler = server::Handler + Send; + pub trait Endpoint : Send + Sync { fn info(&self) -> Option<&EndpointInfo> { None } - fn to_handler(&self, path: EndpointPath) -> Box + Send>; + fn to_handler(&self, path: EndpointPath) -> Box; } - -pub type Endpoints = BTreeMap>; -pub type Handler = server::Handler + Send; diff --git a/dapps/src/handlers/client/fetch_file.rs b/dapps/src/handlers/client/fetch_file.rs new file mode 100644 index 000000000..27b8bbe8e --- /dev/null +++ b/dapps/src/handlers/client/fetch_file.rs @@ -0,0 +1,141 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Hyper Client Handler to Fetch File + +use std::{env, io, fs, fmt}; +use std::path::PathBuf; +use std::sync::mpsc; +use std::time::Duration; +use random_filename; + +use hyper::status::StatusCode; +use hyper::client::{Request, Response, DefaultTransport as HttpStream}; +use hyper::header::Connection; +use hyper::{self, Decoder, Encoder, Next}; + +#[derive(Debug)] +pub enum Error { + NotStarted, + UnexpectedStatus(StatusCode), + IoError(io::Error), + HyperError(hyper::Error), +} + +pub type FetchResult = Result; +pub type OnDone = Box; + +pub struct Fetch { + path: PathBuf, + file: Option, + result: Option, + sender: mpsc::Sender, + on_done: Option, +} + +impl fmt::Debug for Fetch { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result) + } +} + +impl Drop for Fetch { + fn drop(&mut self) { + let res = self.result.take().unwrap_or(Err(Error::NotStarted)); + // Remove file if there was an error + if res.is_err() { + if let Some(file) = self.file.take() { + drop(file); + // Remove file + let _ = fs::remove_file(&self.path); + } + } + // send result + let _ = self.sender.send(res); + if let Some(f) = self.on_done.take() { + f(); + } + } +} + +impl Fetch { + pub fn new(sender: mpsc::Sender, on_done: OnDone) -> Self { + let mut dir = env::temp_dir(); + dir.push(random_filename()); + + Fetch { + path: dir, + file: None, + result: None, + sender: sender, + on_done: Some(on_done), + } + } +} + +impl hyper::client::Handler for Fetch { + fn on_request(&mut self, req: &mut Request) -> Next { + req.headers_mut().set(Connection::close()); + read() + } + + fn on_request_writable(&mut self, _encoder: &mut Encoder) -> Next { + read() + } + + fn on_response(&mut self, res: Response) -> Next { + if *res.status() != StatusCode::Ok { + self.result = Some(Err(Error::UnexpectedStatus(*res.status()))); + return Next::end(); + } + + // Open file to write + match fs::File::create(&self.path) { + Ok(file) => { + self.file = Some(file); + self.result = Some(Ok(self.path.clone())); + read() + }, + Err(err) => { + self.result = Some(Err(Error::IoError(err))); + Next::end() + }, + } + } + + fn on_response_readable(&mut self, decoder: &mut Decoder) -> Next { + match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) { + Ok(0) => Next::end(), + Ok(_) => read(), + Err(e) => match e.kind() { + io::ErrorKind::WouldBlock => Next::read(), + _ => { + self.result = Some(Err(Error::IoError(e))); + Next::end() + } + } + } + } + + fn on_error(&mut self, err: hyper::Error) -> Next { + self.result = Some(Err(Error::HyperError(err))); + Next::remove() + } +} + +fn read() -> Next { + Next::read().timeout(Duration::from_secs(15)) +} diff --git a/dapps/src/handlers/client/mod.rs b/dapps/src/handlers/client/mod.rs new file mode 100644 index 000000000..89532e6f6 --- /dev/null +++ b/dapps/src/handlers/client/mod.rs @@ -0,0 +1,22 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Hyper Client Handlers + +mod fetch_file; + +pub use self::fetch_file::{Fetch, FetchResult, OnDone}; + diff --git a/dapps/src/handlers/content.rs b/dapps/src/handlers/content.rs index b9d8d55d6..092c417ac 100644 --- a/dapps/src/handlers/content.rs +++ b/dapps/src/handlers/content.rs @@ -56,6 +56,10 @@ impl ContentHandler { } } + pub fn html(code: StatusCode, content: String) -> Self { + Self::new(code, content, "text/html".into()) + } + pub fn new(code: StatusCode, content: String, mimetype: String) -> Self { ContentHandler { code: code, diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs new file mode 100644 index 000000000..3ada8fb82 --- /dev/null +++ b/dapps/src/handlers/fetch.rs @@ -0,0 +1,226 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Hyper Server Handler that fetches a file during a request (proxy). + +use std::{fs, fmt}; +use std::path::PathBuf; +use std::sync::mpsc; +use std::time::{Instant, Duration}; + +use hyper::{header, server, Decoder, Encoder, Next, Method, Control, Client}; +use hyper::net::HttpStream; +use hyper::status::StatusCode; + +use handlers::ContentHandler; +use handlers::client::{Fetch, FetchResult}; +use apps::DAPPS_DOMAIN; +use apps::urlhint::GithubApp; +use apps::manifest::Manifest; + +const FETCH_TIMEOUT: u64 = 30; + +enum FetchState { + NotStarted(GithubApp), + Error(ContentHandler), + InProgress { + deadline: Instant, + receiver: mpsc::Receiver + }, + Done(Manifest), +} + +pub trait DappHandler { + type Error: fmt::Debug; + + fn validate_and_install(&self, app: PathBuf) -> Result; + fn done(&self, Option<&Manifest>); +} + +pub struct AppFetcherHandler { + control: Option, + status: FetchState, + client: Option>, + dapp: H, +} + +impl Drop for AppFetcherHandler { + fn drop(&mut self) { + let manifest = match self.status { + FetchState::Done(ref manifest) => Some(manifest), + _ => None, + }; + self.dapp.done(manifest); + } +} + +impl AppFetcherHandler { + + pub fn new( + app: GithubApp, + control: Control, + handler: H) -> Self { + + let client = Client::new().expect("Failed to create a Client"); + AppFetcherHandler { + control: Some(control), + client: Some(client), + status: FetchState::NotStarted(app), + dapp: handler, + } + } + + fn close_client(client: &mut Option>) { + client.take() + .expect("After client is closed we are going into write, hence we can never close it again") + .close(); + } + + + // TODO [todr] https support + fn fetch_app(client: &mut Client, app: &GithubApp, control: Control) -> Result, String> { + let url = try!(app.url().parse().map_err(|e| format!("{:?}", e))); + trace!(target: "dapps", "Fetching from: {:?}", url); + + let (tx, rx) = mpsc::channel(); + let res = client.request(url, Fetch::new(tx, Box::new(move || { + trace!(target: "dapps", "Fetching finished."); + // Ignoring control errors + let _ = control.ready(Next::read()); + }))); + match res { + Ok(_) => Ok(rx), + Err(e) => Err(format!("{:?}", e)), + } + } +} + +impl server::Handler for AppFetcherHandler { + fn on_request(&mut self, request: server::Request) -> Next { + let status = if let FetchState::NotStarted(ref app) = self.status { + Some(match *request.method() { + // Start fetching content + Method::Get => { + trace!(target: "dapps", "Fetching dapp: {:?}", app); + let control = self.control.take().expect("on_request is called only once, thus control is always Some"); + let client = self.client.as_mut().expect("on_request is called before client is closed."); + let fetch = Self::fetch_app(client, app, control); + match fetch { + Ok(receiver) => FetchState::InProgress { + deadline: Instant::now() + Duration::from_secs(FETCH_TIMEOUT), + receiver: receiver, + }, + Err(e) => FetchState::Error(ContentHandler::html( + StatusCode::BadGateway, + format!("

Error starting dapp download.

{}
", e), + )), + } + }, + // or return error + _ => FetchState::Error(ContentHandler::html( + StatusCode::MethodNotAllowed, + "

Only GET requests are allowed.

".into(), + )), + }) + } else { None }; + + if let Some(status) = status { + self.status = status; + } + + Next::read() + } + + fn on_request_readable(&mut self, decoder: &mut Decoder) -> Next { + let (status, next) = match self.status { + // Request may time out + FetchState::InProgress { ref deadline, .. } if *deadline < Instant::now() => { + trace!(target: "dapps", "Fetching dapp failed because of timeout."); + let timeout = ContentHandler::html( + StatusCode::GatewayTimeout, + format!("

Could not fetch app bundle within {} seconds.

", FETCH_TIMEOUT), + ); + Self::close_client(&mut self.client); + (Some(FetchState::Error(timeout)), Next::write()) + }, + FetchState::InProgress { ref receiver, .. } => { + // Check if there is an answer + let rec = receiver.try_recv(); + match rec { + // Unpack and validate + Ok(Ok(path)) => { + trace!(target: "dapps", "Fetching dapp finished. Starting validation."); + Self::close_client(&mut self.client); + // Unpack and verify + let state = match self.dapp.validate_and_install(path.clone()) { + Err(e) => { + trace!(target: "dapps", "Error while validating dapp: {:?}", e); + FetchState::Error(ContentHandler::html( + StatusCode::BadGateway, + format!("

Downloaded bundle does not contain valid app.

{:?}
", e), + )) + }, + Ok(manifest) => FetchState::Done(manifest) + }; + // Remove temporary zip file + let _ = fs::remove_file(path); + (Some(state), Next::write()) + }, + Ok(Err(e)) => { + warn!(target: "dapps", "Unable to fetch new dapp: {:?}", e); + let error = ContentHandler::html( + StatusCode::BadGateway, + "

There was an error when fetching the dapp.

".into(), + ); + (Some(FetchState::Error(error)), Next::write()) + }, + // wait some more + _ => (None, Next::wait()) + } + }, + FetchState::Error(ref mut handler) => (None, handler.on_request_readable(decoder)), + _ => (None, Next::write()), + }; + + if let Some(status) = status { + self.status = status; + } + + next + } + + fn on_response(&mut self, res: &mut server::Response) -> Next { + match self.status { + FetchState::Done(ref manifest) => { + trace!(target: "dapps", "Fetching dapp finished. Redirecting to {}", manifest.id); + res.set_status(StatusCode::Found); + // TODO [todr] should detect if its using nice-urls + res.headers_mut().set(header::Location(format!("http://{}{}", manifest.id, DAPPS_DOMAIN))); + Next::write() + }, + FetchState::Error(ref mut handler) => handler.on_response(res), + _ => Next::end(), + } + } + + fn on_response_writable(&mut self, encoder: &mut Encoder) -> Next { + match self.status { + FetchState::Error(ref mut handler) => handler.on_response_writable(encoder), + _ => Next::end(), + } + } +} + diff --git a/dapps/src/handlers/mod.rs b/dapps/src/handlers/mod.rs index 5fa9fda95..85a8bd439 100644 --- a/dapps/src/handlers/mod.rs +++ b/dapps/src/handlers/mod.rs @@ -20,11 +20,14 @@ mod auth; mod echo; mod content; mod redirect; +mod fetch; +pub mod client; pub use self::auth::AuthRequiredHandler; pub use self::echo::EchoHandler; pub use self::content::ContentHandler; pub use self::redirect::Redirection; +pub use self::fetch::{AppFetcherHandler, DappHandler}; use url::Url; use hyper::{server, header, net, uri}; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index d9fa06591..49940080f 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -50,12 +50,15 @@ extern crate hyper; extern crate unicase; extern crate serde; extern crate serde_json; +extern crate zip; +extern crate rand; extern crate jsonrpc_core; extern crate jsonrpc_http_server; extern crate parity_dapps; extern crate ethcore_rpc; -extern crate ethcore_util; +extern crate ethcore_util as util; extern crate mime_guess; +extern crate rustc_serialize; mod endpoint; mod apps; @@ -121,6 +124,7 @@ impl Server { fn start_http(addr: &SocketAddr, authorization: A, handler: Arc, dapps_path: String) -> Result { let panic_handler = Arc::new(Mutex::new(None)); let authorization = Arc::new(authorization); + let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::default()); let endpoints = Arc::new(apps::all_endpoints(dapps_path)); let special = Arc::new({ let mut special = HashMap::new(); @@ -132,8 +136,10 @@ impl Server { let bind_address = format!("{}", addr); try!(hyper::Server::http(addr)) - .handle(move |_| router::Router::new( + .handle(move |ctrl| router::Router::new( + ctrl, apps::main_page(), + apps_fetcher.clone(), endpoints.clone(), special.clone(), authorization.clone(), @@ -182,3 +188,11 @@ impl From for ServerError { } } } + +/// Random filename +pub fn random_filename() -> String { + use ::rand::Rng; + let mut rng = ::rand::OsRng::new().unwrap(); + rng.gen_ascii_chars().take(12).collect() +} + diff --git a/dapps/src/router/host_validation.rs b/dapps/src/router/host_validation.rs index 3b065cd0c..62813500f 100644 --- a/dapps/src/router/host_validation.rs +++ b/dapps/src/router/host_validation.rs @@ -16,13 +16,12 @@ use DAPPS_DOMAIN; -use hyper::server; +use hyper::{server, header}; use hyper::net::HttpStream; use jsonrpc_http_server::{is_host_header_valid}; use handlers::ContentHandler; - pub fn is_valid(request: &server::Request, bind_address: &str, endpoints: Vec) -> bool { let mut endpoints = endpoints.into_iter() .map(|endpoint| format!("{}{}", endpoint, DAPPS_DOMAIN)) @@ -31,7 +30,13 @@ pub fn is_valid(request: &server::Request, bind_address: &str, endpo endpoints.push(bind_address.replace("127.0.0.1", "localhost").into()); endpoints.push(bind_address.into()); - is_host_header_valid(request, &endpoints) + let header_valid = is_host_header_valid(request, &endpoints); + + match (header_valid, request.headers().get::()) { + (true, _) => true, + (_, Some(host)) => host.hostname.ends_with(DAPPS_DOMAIN), + _ => false, + } } pub fn host_invalid_response() -> Box + Send> { diff --git a/dapps/src/router/mod.rs b/dapps/src/router/mod.rs index bdd5fd291..3dad8250b 100644 --- a/dapps/src/router/mod.rs +++ b/dapps/src/router/mod.rs @@ -24,9 +24,10 @@ use DAPPS_DOMAIN; use std::sync::Arc; use std::collections::HashMap; use url::{Url, Host}; -use hyper::{self, server, Next, Encoder, Decoder}; +use hyper::{self, server, Next, Encoder, Decoder, Control}; use hyper::net::HttpStream; use apps; +use apps::fetcher::AppFetcher; use endpoint::{Endpoint, Endpoints, EndpointPath}; use handlers::{Redirection, extract_url}; use self::auth::{Authorization, Authorized}; @@ -41,8 +42,10 @@ pub enum SpecialEndpoint { } pub struct Router { + control: Option, main_page: &'static str, endpoints: Arc, + fetch: Arc, special: Arc>>, authorization: Arc, bind_address: String, @@ -78,6 +81,11 @@ impl server::Handler for Router { (Some(ref path), _) if self.endpoints.contains_key(&path.app_id) => { self.endpoints.get(&path.app_id).unwrap().to_handler(path.clone()) }, + // Try to resolve and fetch dapp + (Some(ref path), _) if self.fetch.contains(&path.app_id) => { + let control = self.control.take().expect("on_request is called only once, thus control is always defined."); + self.fetch.to_handler(path.clone(), control) + }, // Redirection to main page _ if *req.method() == hyper::method::Method::Get => { Redirection::new(self.main_page) @@ -110,7 +118,9 @@ impl server::Handler for Router { impl Router { pub fn new( + control: Control, main_page: &'static str, + app_fetcher: Arc, endpoints: Arc, special: Arc>>, authorization: Arc, @@ -119,8 +129,10 @@ impl Router { let handler = special.get(&SpecialEndpoint::Rpc).unwrap().to_handler(EndpointPath::default()); Router { + control: Some(control), main_page: main_page, endpoints: endpoints, + fetch: app_fetcher, special: special, authorization: authorization, bind_address: bind_address,