Fetchable dapps (#1949)

* Fetching dapp from github.

* Unpacking dapp

* Removing hardcodes

* Proper Host validation

* Randomizing paths

* Splitting into files

* Serving donwloaded apps from different path

* Extracting URLHint to separate module

* Whitespace and docs

Cargo.lock

@@ -286,6 +286,7 @@ dependencies = [
  "parity-dapps-home 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
  "parity-dapps-status 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
  "parity-dapps-wallet 1.4.0 (git+https://github.com/ethcore/parity-ui.git)",
+ "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_codegen 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -293,6 +294,7 @@ dependencies = [
  "syntex 0.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "zip 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -549,6 +551,15 @@ dependencies = [
  "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "flate2"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "gcc"
 version = "0.3.28"
@@ -780,6 +791,15 @@ dependencies = [
  "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "mio"
 version = "0.5.1"
@@ -838,6 +858,16 @@ dependencies = [
  "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "msdos_time"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "nanomsg"
 version = "0.5.1"
@@ -1081,6 +1111,11 @@ dependencies = [
  "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "podio"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "primal"
 version = "0.2.3"
@@ -1587,6 +1622,17 @@ dependencies = [
  "xml-rs 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "zip"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "msdos_time 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [metadata]
 "checksum aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67077478f0a03952bed2e6786338d400d40c25e9836e08ad50af96607317fd03"
 "checksum ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f46cd5b1d660c938e3f92dfe7a73d832b3281479363dd0cd9c1c2fbf60f7962"
@@ -1612,6 +1658,7 @@ dependencies = [
 "checksum elastic-array 0.4.0 (git+https://github.com/ethcore/elastic-array)" = "<none>"
 "checksum env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aba65b63ffcc17ffacd6cf5aa843da7c5a25e3bd4bbe0b7def8b214e411250e5"
 "checksum eth-secp256k1 0.5.4 (git+https://github.com/ethcore/rust-secp256k1)" = "<none>"
+"checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
 "checksum gcc 0.3.28 (registry+https://github.com/rust-lang/crates.io-index)" = "3da3a2cbaeb01363c8e3704fd9fd0eb2ceb17c6f27abd4c1ef040fb57d20dc79"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
 "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
@@ -1637,10 +1684,12 @@ dependencies = [
 "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
 "checksum mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a74cc2587bf97c49f3f5bab62860d6abf3902ca73b66b51d9b049fbdcd727bd2"
 "checksum mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e50bf542f81754ef69e5cea856946a3819f7c09ea97b4903c8bc8a89f74e7b6"
+"checksum miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d1f4d337a01c32e1f2122510fed46393d53ca35a7f429cb0450abaedfa3ed54"
 "checksum mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)" = "<none>"
 "checksum mio 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a637d1ca14eacae06296a008fa7ad955347e34efcb5891cfd8ba05491a37907e"
 "checksum mio 0.6.0-dev (git+https://github.com/carllerche/mio?rev=62ec763c9cc34d8a452ed0392c575c50ddd5fc8d)" = "<none>"
 "checksum miow 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d5bfc6782530ac8ace97af10a540054a37126b63b0702ddaaa243b73b5745b9a"
+"checksum msdos_time 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c04b68cc63a8480fb2550343695f7be72effdec953a9d4508161c3e69041c7d8"
 "checksum nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)" = "<none>"
 "checksum nanomsg-sys 0.5.0 (git+https://github.com/ethcore/nanomsg.rs.git)" = "<none>"
 "checksum net2 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "6a816012ca11cb47009693c1e0c6130e26d39e4d97ee2a13c50e868ec83e3204"
@@ -1668,6 +1717,7 @@ dependencies = [
 "checksum phf_codegen 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8af7ae7c3f75a502292b491e5cc0a1f69e3407744abe6e57e2a3b712bb82f01d"
 "checksum phf_generator 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "db005608fd99800c8c74106a7c894cf582055b689aa14a79462cefdcb7dc1cc3"
 "checksum phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "fee4d039930e4f45123c9b15976cf93a499847b6483dc09c42ea0ec4940f2aa6"
+"checksum podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e5422a1ee1bc57cc47ae717b0137314258138f38fd5f3cea083f43a9725383a0"
 "checksum primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0e31b86efadeaeb1235452171a66689682783149a6249ff334a2c5d8218d00a4"
 "checksum primal-bit 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "464a91febc06166783d4f5ba3577b5ed8dda8e421012df80bfe48a971ed7be8f"
 "checksum primal-check 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "647c81b67bb9551a7b88d0bcd785ac35b7d0bf4b2f358683d7c2375d04daec51"
@@ -1732,3 +1782,4 @@ dependencies = [
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xml-rs 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "65e74b96bd3179209dc70a980da6df843dff09e46eee103a0376c0949257e3ef"
 "checksum xmltree 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "472a9d37c7c53ab2391161df5b89b1f3bf76dab6ab150d7941ecbdd832282082"
+"checksum zip 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3ceb33a75b3d0608942302eed325b59d2c3ed777cc6c01627ae14e5697c6a31c"

dapps/Cargo.toml

@@ -9,6 +9,7 @@ build = "build.rs"
 [lib]
 
 [dependencies]
+rand = "0.3.14"
 log = "0.3"
 jsonrpc-core = "2.1"
 jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc-http-server.git" }
@@ -19,6 +20,7 @@ rustc-serialize = "0.3"
 serde = "0.7.0"
 serde_json = "0.7.0"
 serde_macros = { version = "0.7.0", optional = true }
+zip = { version = "0.1", default-features = false }
 ethcore-rpc = { path = "../rpc" }
 ethcore-util = { path = "../util" }
 parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }

dapps/src/api/types.rs.in

@@ -16,7 +16,7 @@
 
 use endpoint::EndpointInfo;
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub struct App {
 	pub id: String,
 	pub name: String,
@@ -41,6 +41,18 @@ impl App {
 	}
 }
 
+impl Into<EndpointInfo> for App {
+	fn into(self) -> EndpointInfo {
+		EndpointInfo {
+			name: self.name,
+			description: self.description,
+			version: self.version,
+			author: self.author,
+			icon_url: self.icon_url,
+		}
+	}
+}
+
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct ApiError {
 	pub code: String,

dapps/src/apps/fetcher.rs

@@ -0,0 +1,298 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Fetchable Dapps support.
+//! Manages downloaded (cached) Dapps and downloads them when necessary.
+//! Uses `URLHint` to resolve addresses into Dapps bundle file location.
+
+use zip;
+use std::{fs, env};
+use std::io::{self, Read, Write};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::collections::HashMap;
+
+use hyper::Control;
+use hyper::status::StatusCode;
+
+use random_filename;
+use util::Mutex;
+use page::LocalPageEndpoint;
+use handlers::{ContentHandler, AppFetcherHandler, DappHandler};
+use endpoint::{Endpoint, EndpointPath, Handler};
+use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
+use apps::urlhint::{URLHintContract, URLHint};
+
+enum AppStatus {
+	Fetching,
+	Ready(LocalPageEndpoint),
+}
+
+pub struct AppFetcher<R: URLHint = URLHintContract> {
+	dapps_path: PathBuf,
+	resolver: R,
+	dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
+}
+
+impl<R: URLHint> Drop for AppFetcher<R> {
+	fn drop(&mut self) {
+		// Clear cache path
+		let _ = fs::remove_dir_all(&self.dapps_path);
+	}
+}
+
+impl Default for AppFetcher<URLHintContract> {
+	fn default() -> Self {
+		AppFetcher::new(URLHintContract)
+	}
+}
+
+impl<R: URLHint> AppFetcher<R> {
+
+	pub fn new(resolver: R) -> Self {
+		let mut dapps_path = env::temp_dir();
+		dapps_path.push(random_filename());
+
+		AppFetcher {
+			dapps_path: dapps_path,
+			resolver: resolver,
+			dapps: Arc::new(Mutex::new(HashMap::new())),
+		}
+	}
+
+	#[cfg(test)]
+	fn set_status(&self, app_id: &str, status: AppStatus) {
+		self.dapps.lock().insert(app_id.to_owned(), status);
+	}
+
+	pub fn contains(&self, app_id: &str) -> bool {
+		let dapps = self.dapps.lock();
+		match dapps.get(app_id) {
+			// Check if we already have the app
+			Some(_) => true,
+			// fallback to resolver
+			None => self.resolver.resolve(app_id).is_some(),
+		}
+	}
+
+	pub fn to_handler(&self, path: EndpointPath, control: Control) -> Box<Handler> {
+		let mut dapps = self.dapps.lock();
+		let app_id = path.app_id.clone();
+
+		let (new_status, handler) = {
+			let status = dapps.get(&app_id);
+			match status {
+				// Just server dapp
+				Some(&AppStatus::Ready(ref endpoint)) => {
+					(None, endpoint.to_handler(path))
+				},
+				// App is already being fetched
+				Some(&AppStatus::Fetching) => {
+					(None, Box::new(ContentHandler::html(
+						StatusCode::ServiceUnavailable,
+						"<h1>This dapp is already being downloaded.</h1>".into()
+					)) as Box<Handler>)
+				},
+				// We need to start fetching app
+				None => {
+					// TODO [todr] Keep only last N dapps available!
+					let app = self.resolver.resolve(&app_id).expect("to_handler is called only when `contains` returns true.");
+					(Some(AppStatus::Fetching), Box::new(AppFetcherHandler::new(
+						app,
+						control,
+						DappInstaller {
+							dapp_id: app_id.clone(),
+							dapps_path: self.dapps_path.clone(),
+							dapps: self.dapps.clone(),
+						}
+					)) as Box<Handler>)
+				},
+			}
+		};
+
+		if let Some(status) = new_status {
+			dapps.insert(app_id, status);
+		}
+
+		handler
+	}
+}
+
+#[derive(Debug)]
+pub enum ValidationError {
+	ManifestNotFound,
+	ManifestSerialization(String),
+	Io(io::Error),
+	Zip(zip::result::ZipError),
+}
+
+impl From<io::Error> for ValidationError {
+	fn from(err: io::Error) -> Self {
+		ValidationError::Io(err)
+	}
+}
+
+impl From<zip::result::ZipError> for ValidationError {
+	fn from(err: zip::result::ZipError) -> Self {
+		ValidationError::Zip(err)
+	}
+}
+
+struct DappInstaller {
+	dapp_id: String,
+	dapps_path: PathBuf,
+	dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
+}
+
+impl DappInstaller {
+	fn find_manifest(zip: &mut zip::ZipArchive<fs::File>) -> Result<(Manifest, PathBuf), ValidationError> {
+		for i in 0..zip.len() {
+			let mut file = try!(zip.by_index(i));
+
+			if !file.name().ends_with(MANIFEST_FILENAME) {
+				continue;
+			}
+
+			// try to read manifest
+			let mut manifest = String::new();
+			let manifest = file
+				.read_to_string(&mut manifest).ok()
+				.and_then(|_| deserialize_manifest(manifest).ok());
+
+			if let Some(manifest) = manifest {
+				let mut manifest_location = PathBuf::from(file.name());
+				manifest_location.pop(); // get rid of filename
+				return Ok((manifest, manifest_location));
+			}
+		}
+
+		Err(ValidationError::ManifestNotFound)
+	}
+
+	fn dapp_target_path(&self, manifest: &Manifest) -> PathBuf {
+		let mut target = self.dapps_path.clone();
+		target.push(&manifest.id);
+		target
+	}
+}
+
+impl DappHandler for DappInstaller {
+	type Error = ValidationError;
+
+	fn validate_and_install(&self, app_path: PathBuf) -> Result<Manifest, ValidationError> {
+		trace!(target: "dapps", "Opening dapp bundle at {:?}", app_path);
+		// TODO [ToDr] Validate file hash
+		let file = try!(fs::File::open(app_path));
+		// Unpack archive
+		let mut zip = try!(zip::ZipArchive::new(file));
+		// First find manifest file
+		let (mut manifest, manifest_dir) = try!(Self::find_manifest(&mut zip));
+		// Overwrite id to match hash
+		manifest.id = self.dapp_id.clone();
+
+		let target = self.dapp_target_path(&manifest);
+
+		// Remove old directory
+		if target.exists() {
+			warn!(target: "dapps", "Overwriting existing dapp: {}", manifest.id);
+			try!(fs::remove_dir_all(target.clone()));
+		}
+
+		// Unpack zip
+		for i in 0..zip.len() {
+			let mut file = try!(zip.by_index(i));
+			// TODO [todr] Check if it's consistent on windows.
+			let is_dir = file.name().chars().rev().next() == Some('/');
+
+			let file_path = PathBuf::from(file.name());
+			let location_in_manifest_base = file_path.strip_prefix(&manifest_dir);
+			// Create files that are inside manifest directory
+			if let Ok(location_in_manifest_base) = location_in_manifest_base {
+				let p = target.join(location_in_manifest_base);
+				// Check if it's a directory
+				if is_dir {
+					try!(fs::create_dir_all(p));
+				} else {
+					let mut target = try!(fs::File::create(p));
+					try!(io::copy(&mut file, &mut target));
+				}
+			}
+		}
+
+		// Write manifest
+		let manifest_str = try!(serialize_manifest(&manifest).map_err(ValidationError::ManifestSerialization));
+		let manifest_path = target.join(MANIFEST_FILENAME);
+		let mut manifest_file = try!(fs::File::create(manifest_path));
+		try!(manifest_file.write_all(manifest_str.as_bytes()));
+
+		// Return modified app manifest
+		Ok(manifest)
+	}
+
+	fn done(&self, manifest: Option<&Manifest>) {
+		let mut dapps = self.dapps.lock();
+		match manifest {
+			Some(manifest) => {
+				let path = self.dapp_target_path(manifest);
+				let app = LocalPageEndpoint::new(path, manifest.clone().into());
+				dapps.insert(self.dapp_id.clone(), AppStatus::Ready(app));
+			},
+			// In case of error
+			None => {
+				dapps.remove(&self.dapp_id);
+			},
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::path::PathBuf;
+	use super::{AppFetcher, AppStatus};
+	use apps::urlhint::{GithubApp, URLHint};
+	use endpoint::EndpointInfo;
+	use page::LocalPageEndpoint;
+
+	struct FakeResolver;
+	impl URLHint for FakeResolver {
+		fn resolve(&self, _app_id: &str) -> Option<GithubApp> {
+			None
+		}
+	}
+
+	#[test]
+	fn should_true_if_contains_the_app() {
+		// given
+		let fetcher = AppFetcher::new(FakeResolver);
+		let handler = LocalPageEndpoint::new(PathBuf::from("/tmp/test"), EndpointInfo {
+			name: "fake".into(),
+			description: "".into(),
+			version: "".into(),
+			author: "".into(),
+			icon_url: "".into(),
+		});
+
+		// when
+		fetcher.set_status("test", AppStatus::Ready(handler));
+		fetcher.set_status("test2", AppStatus::Fetching);
+
+		// then
+		assert_eq!(fetcher.contains("test"), true);
+		assert_eq!(fetcher.contains("test2"), true);
+		assert_eq!(fetcher.contains("test3"), false);
+	}
+}
+

dapps/src/apps/fs.rs

@@ -14,14 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use serde_json;
 use std::io;
 use std::io::Read;
 use std::fs;
 use std::path::PathBuf;
 use page::LocalPageEndpoint;
 use endpoint::{Endpoints, EndpointInfo};
-use api::App;
+use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest};
 
 struct LocalDapp {
 	id: String,
@@ -73,7 +72,7 @@ fn local_dapps(dapps_path: String) -> Vec<LocalDapp> {
 }
 
 fn read_manifest(name: &str, mut path: PathBuf) -> EndpointInfo {
-	path.push("manifest.json");
+	path.push(MANIFEST_FILENAME);
 
 	fs::File::open(path.clone())
 		.map_err(|e| format!("{:?}", e))
@@ -82,15 +81,9 @@ fn read_manifest(name: &str, mut path: PathBuf) -> EndpointInfo {
 			let mut s = String::new();
 			try!(f.read_to_string(&mut s).map_err(|e| format!("{:?}", e)));
 			// Try to deserialize manifest
-			serde_json::from_str::<App>(&s).map_err(|e| format!("{:?}", e))
+			deserialize_manifest(s)
-		})
-		.map(|app| EndpointInfo {
-			name: app.name,
-			description: app.description,
-			version: app.version,
-			author: app.author,
-			icon_url: app.icon_url,
 		})
+		.map(Into::into)
 		.unwrap_or_else(|e| {
 			warn!(target: "dapps", "Cannot read manifest file at: {:?}. Error: {:?}", path, e);
 

dapps/src/apps/manifest.rs

@@ -0,0 +1,29 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use serde_json;
+pub use api::App as Manifest;
+
+pub const MANIFEST_FILENAME: &'static str = "manifest.json";
+
+pub fn deserialize_manifest(manifest: String) -> Result<Manifest, String> {
+	serde_json::from_str::<Manifest>(&manifest).map_err(|e| format!("{:?}", e))
+	// TODO [todr] Manifest validation (especialy: id (used as path))
+}
+
+pub fn serialize_manifest(manifest: &Manifest) -> Result<String, String> {
+	serde_json::to_string_pretty(manifest).map_err(|e| format!("{:?}", e))
+}

dapps/src/apps/mod.rs

@@ -20,6 +20,9 @@ use proxypac::ProxyPac;
 use parity_dapps::WebApp;
 
 mod fs;
+pub mod urlhint;
+pub mod fetcher;
+pub mod manifest;
 
 extern crate parity_dapps_status;
 extern crate parity_dapps_home;

dapps/src/apps/urlhint.rs

@@ -0,0 +1,104 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use rustc_serialize::hex::ToHex;
+
+use util::{Address, FromHex};
+
+const COMMIT_LEN: usize = 20;
+
+#[derive(Debug)]
+pub struct GithubApp {
+	pub account: String,
+	pub repo: String,
+	pub commit: [u8;COMMIT_LEN],
+	pub owner: Address,
+}
+
+impl GithubApp {
+	pub fn url(&self) -> String {
+		// format!("https://github.com/{}/{}/archive/{}.zip", self.account, self.repo, self.commit.to_hex())
+		format!("http://github.todr.me/{}/{}/zip/{}", self.account, self.repo, self.commit.to_hex())
+	}
+
+	fn commit(bytes: &[u8]) -> Option<[u8;COMMIT_LEN]> {
+		if bytes.len() < COMMIT_LEN {
+			return None;
+		}
+
+		let mut commit = [0; COMMIT_LEN];
+		for i in 0..COMMIT_LEN {
+			commit[i] = bytes[i];
+		}
+
+		Some(commit)
+	}
+}
+
+pub trait URLHint {
+	fn resolve(&self, app_id: &str) -> Option<GithubApp>;
+}
+
+pub struct URLHintContract;
+
+impl URLHint for URLHintContract {
+	fn resolve(&self, app_id: &str) -> Option<GithubApp> {
+		// TODO [todr] use GithubHint contract to check the details
+		// For now we are just accepting patterns: <commithash>.<repo>.<account>.parity
+		let mut app_parts = app_id.split('.');
+
+		let hash = app_parts.next()
+			.and_then(|h| h.from_hex().ok())
+			.and_then(|h| GithubApp::commit(&h));
+		let repo = app_parts.next();
+		let account = app_parts.next();
+
+		match (hash, repo, account) {
+			(Some(hash), Some(repo), Some(account)) => {
+				Some(GithubApp {
+					account: account.into(),
+					repo: repo.into(),
+					commit: hash,
+					owner: Address::default(),
+				})
+			},
+			_ => None,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::GithubApp;
+	use util::Address;
+
+	#[test]
+	fn should_return_valid_url() {
+		// given
+		let app = GithubApp {
+			account: "test".into(),
+			repo: "xyz".into(),
+			commit: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
+			owner: Address::default(),
+		};
+
+		// when
+		let url = app.url();
+
+		// then
+		assert_eq!(url, "http://github.todr.me/test/xyz/zip/000102030405060708090a0b0c0d0e0f10111213".to_owned());
+	}
+}

dapps/src/endpoint.rs

@@ -35,11 +35,11 @@ pub struct EndpointInfo {
 	pub icon_url: String,
 }
 
+pub type Endpoints = BTreeMap<String, Box<Endpoint>>;
+pub type Handler = server::Handler<net::HttpStream> + Send;
+
 pub trait Endpoint : Send + Sync {
 	fn info(&self) -> Option<&EndpointInfo> { None }
 
-	fn to_handler(&self, path: EndpointPath) -> Box<server::Handler<net::HttpStream> + Send>;
+	fn to_handler(&self, path: EndpointPath) -> Box<Handler>;
 }
-
-pub type Endpoints = BTreeMap<String, Box<Endpoint>>;
-pub type Handler = server::Handler<net::HttpStream> + Send;

dapps/src/handlers/client/fetch_file.rs

@@ -0,0 +1,141 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hyper Client Handler to Fetch File
+
+use std::{env, io, fs, fmt};
+use std::path::PathBuf;
+use std::sync::mpsc;
+use std::time::Duration;
+use random_filename;
+
+use hyper::status::StatusCode;
+use hyper::client::{Request, Response, DefaultTransport as HttpStream};
+use hyper::header::Connection;
+use hyper::{self, Decoder, Encoder, Next};
+
+#[derive(Debug)]
+pub enum Error {
+	NotStarted,
+	UnexpectedStatus(StatusCode),
+	IoError(io::Error),
+	HyperError(hyper::Error),
+}
+
+pub type FetchResult = Result<PathBuf, Error>;
+pub type OnDone = Box<Fn() + Send>;
+
+pub struct Fetch {
+	path: PathBuf,
+	file: Option<fs::File>,
+	result: Option<FetchResult>,
+	sender: mpsc::Sender<FetchResult>,
+	on_done: Option<OnDone>,
+}
+
+impl fmt::Debug for Fetch {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result)
+	}
+}
+
+impl Drop for Fetch {
+    fn drop(&mut self) {
+		let res = self.result.take().unwrap_or(Err(Error::NotStarted));
+		// Remove file if there was an error
+		if res.is_err() {
+			if let Some(file) = self.file.take() {
+				drop(file);
+				// Remove file
+				let _ = fs::remove_file(&self.path);
+			}
+		}
+		// send result
+		let _ = self.sender.send(res);
+		if let Some(f) = self.on_done.take() {
+			f();
+		}
+    }
+}
+
+impl Fetch {
+	pub fn new(sender: mpsc::Sender<FetchResult>, on_done: OnDone) -> Self {
+		let mut dir = env::temp_dir();
+		dir.push(random_filename());
+
+		Fetch {
+			path: dir,
+			file: None,
+			result: None,
+			sender: sender,
+			on_done: Some(on_done),
+		}
+	}
+}
+
+impl hyper::client::Handler<HttpStream> for Fetch {
+    fn on_request(&mut self, req: &mut Request) -> Next {
+        req.headers_mut().set(Connection::close());
+        read()
+    }
+
+    fn on_request_writable(&mut self, _encoder: &mut Encoder<HttpStream>) -> Next {
+        read()
+    }
+
+    fn on_response(&mut self, res: Response) -> Next {
+		if *res.status() != StatusCode::Ok {
+			self.result = Some(Err(Error::UnexpectedStatus(*res.status())));
+			return Next::end();
+		}
+
+		// Open file to write
+		match fs::File::create(&self.path) {
+			Ok(file) => {
+				self.file = Some(file);
+				self.result = Some(Ok(self.path.clone()));
+				read()
+			},
+			Err(err) => {
+				self.result = Some(Err(Error::IoError(err)));
+				Next::end()
+			},
+		}
+    }
+
+    fn on_response_readable(&mut self, decoder: &mut Decoder<HttpStream>) -> Next {
+        match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) {
+            Ok(0) => Next::end(),
+            Ok(_) => read(),
+            Err(e) => match e.kind() {
+                io::ErrorKind::WouldBlock => Next::read(),
+                _ => {
+					self.result = Some(Err(Error::IoError(e)));
+                    Next::end()
+                }
+            }
+        }
+    }
+
+    fn on_error(&mut self, err: hyper::Error) -> Next {
+		self.result = Some(Err(Error::HyperError(err)));
+        Next::remove()
+    }
+}
+
+fn read() -> Next {
+    Next::read().timeout(Duration::from_secs(15))
+}

dapps/src/handlers/client/mod.rs

@@ -0,0 +1,22 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hyper Client Handlers
+
+mod fetch_file;
+
+pub use self::fetch_file::{Fetch, FetchResult, OnDone};
+

dapps/src/handlers/content.rs

@@ -56,6 +56,10 @@ impl ContentHandler {
 		}
 	}
 
+	pub fn html(code: StatusCode, content: String) -> Self {
+		Self::new(code, content, "text/html".into())
+	}
+
 	pub fn new(code: StatusCode, content: String, mimetype: String) -> Self {
 		ContentHandler {
 			code: code,

dapps/src/handlers/fetch.rs

@@ -0,0 +1,226 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hyper Server Handler that fetches a file during a request (proxy).
+
+use std::{fs, fmt};
+use std::path::PathBuf;
+use std::sync::mpsc;
+use std::time::{Instant, Duration};
+
+use hyper::{header, server, Decoder, Encoder, Next, Method, Control, Client};
+use hyper::net::HttpStream;
+use hyper::status::StatusCode;
+
+use handlers::ContentHandler;
+use handlers::client::{Fetch, FetchResult};
+use apps::DAPPS_DOMAIN;
+use apps::urlhint::GithubApp;
+use apps::manifest::Manifest;
+
+const FETCH_TIMEOUT: u64 = 30;
+
+enum FetchState {
+	NotStarted(GithubApp),
+	Error(ContentHandler),
+	InProgress {
+		deadline: Instant,
+		receiver: mpsc::Receiver<FetchResult>
+	},
+	Done(Manifest),
+}
+
+pub trait DappHandler {
+	type Error: fmt::Debug;
+
+	fn validate_and_install(&self, app: PathBuf) -> Result<Manifest, Self::Error>;
+	fn done(&self, Option<&Manifest>);
+}
+
+pub struct AppFetcherHandler<H: DappHandler> {
+	control: Option<Control>,
+	status: FetchState,
+	client: Option<Client<Fetch>>,
+	dapp: H,
+}
+
+impl<H: DappHandler> Drop for AppFetcherHandler<H> {
+	fn drop(&mut self) {
+		let manifest = match self.status {
+			FetchState::Done(ref manifest) => Some(manifest),
+			_ => None,
+		};
+		self.dapp.done(manifest);
+	}
+}
+
+impl<H: DappHandler> AppFetcherHandler<H> {
+
+	pub fn new(
+		app: GithubApp,
+		control: Control,
+		handler: H) -> Self {
+
+		let client = Client::new().expect("Failed to create a Client");
+		AppFetcherHandler {
+			control: Some(control),
+			client: Some(client),
+			status: FetchState::NotStarted(app),
+			dapp: handler,
+		}
+	}
+
+	fn close_client(client: &mut Option<Client<Fetch>>) {
+		client.take()
+			.expect("After client is closed we are going into write, hence we can never close it again")
+			.close();
+	}
+
+
+	// TODO [todr] https support
+	fn fetch_app(client: &mut Client<Fetch>, app: &GithubApp, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
+		let url = try!(app.url().parse().map_err(|e| format!("{:?}", e)));
+		trace!(target: "dapps", "Fetching from: {:?}", url);
+
+		let (tx, rx) = mpsc::channel();
+		let res = client.request(url, Fetch::new(tx, Box::new(move || {
+			trace!(target: "dapps", "Fetching finished.");
+			// Ignoring control errors
+			let _ = control.ready(Next::read());
+		})));
+		match res {
+			Ok(_) => Ok(rx),
+			Err(e) => Err(format!("{:?}", e)),
+		}
+	}
+}
+
+impl<H: DappHandler> server::Handler<HttpStream> for AppFetcherHandler<H> {
+	fn on_request(&mut self, request: server::Request<HttpStream>) -> Next {
+		let status = if let FetchState::NotStarted(ref app) = self.status {
+			Some(match *request.method() {
+				// Start fetching content
+				Method::Get => {
+					trace!(target: "dapps", "Fetching dapp: {:?}", app);
+					let control = self.control.take().expect("on_request is called only once, thus control is always Some");
+					let client = self.client.as_mut().expect("on_request is called before client is closed.");
+					let fetch = Self::fetch_app(client, app, control);
+					match fetch {
+						Ok(receiver) => FetchState::InProgress {
+							deadline: Instant::now() + Duration::from_secs(FETCH_TIMEOUT),
+							receiver: receiver,
+						},
+						Err(e) => FetchState::Error(ContentHandler::html(
+							StatusCode::BadGateway,
+							format!("<h1>Error starting dapp download.</h1><pre>{}</pre>", e),
+						)),
+					}
+				},
+				// or return error
+				_ => FetchState::Error(ContentHandler::html(
+					StatusCode::MethodNotAllowed,
+					"<h1>Only <code>GET</code> requests are allowed.</h1>".into(),
+				)),
+			})
+		} else { None };
+
+		if let Some(status) = status {
+			self.status = status;
+		}
+
+		Next::read()
+	}
+
+	fn on_request_readable(&mut self, decoder: &mut Decoder<HttpStream>) -> Next {
+		let (status, next) = match self.status {
+			// Request may time out
+			FetchState::InProgress { ref deadline, .. } if *deadline < Instant::now() => {
+				trace!(target: "dapps", "Fetching dapp failed because of timeout.");
+				let timeout = ContentHandler::html(
+					StatusCode::GatewayTimeout,
+					format!("<h1>Could not fetch app bundle within {} seconds.</h1>", FETCH_TIMEOUT),
+					);
+				Self::close_client(&mut self.client);
+				(Some(FetchState::Error(timeout)), Next::write())
+			},
+			FetchState::InProgress { ref receiver, .. } => {
+				// Check if there is an answer
+				let rec = receiver.try_recv();
+				match rec {
+					// Unpack and validate
+					Ok(Ok(path)) => {
+						trace!(target: "dapps", "Fetching dapp finished. Starting validation.");
+						Self::close_client(&mut self.client);
+						// Unpack and verify
+						let state = match self.dapp.validate_and_install(path.clone()) {
+							Err(e) => {
+								trace!(target: "dapps", "Error while validating dapp: {:?}", e);
+								FetchState::Error(ContentHandler::html(
+									StatusCode::BadGateway,
+									format!("<h1>Downloaded bundle does not contain valid app.</h1><pre>{:?}</pre>", e),
+								))
+							},
+							Ok(manifest) => FetchState::Done(manifest)
+						};
+						// Remove temporary zip file
+						let _ = fs::remove_file(path);
+						(Some(state), Next::write())
+					},
+					Ok(Err(e)) => {
+						warn!(target: "dapps", "Unable to fetch new dapp: {:?}", e);
+						let error = ContentHandler::html(
+							StatusCode::BadGateway,
+							"<h1>There was an error when fetching the dapp.</h1>".into(),
+						);
+						(Some(FetchState::Error(error)), Next::write())
+					},
+					// wait some more
+					_ => (None, Next::wait())
+				}
+			},
+			FetchState::Error(ref mut handler) => (None, handler.on_request_readable(decoder)),
+			_ => (None, Next::write()),
+		};
+
+		if let Some(status) = status {
+			self.status = status;
+		}
+
+		next
+	}
+
+	fn on_response(&mut self, res: &mut server::Response) -> Next {
+		match self.status {
+			FetchState::Done(ref manifest) => {
+				trace!(target: "dapps", "Fetching dapp finished. Redirecting to {}", manifest.id);
+				res.set_status(StatusCode::Found);
+				// TODO [todr] should detect if its using nice-urls
+				res.headers_mut().set(header::Location(format!("http://{}{}", manifest.id, DAPPS_DOMAIN)));
+				Next::write()
+			},
+			FetchState::Error(ref mut handler) => handler.on_response(res),
+			_ => Next::end(),
+		}
+	}
+
+	fn on_response_writable(&mut self, encoder: &mut Encoder<HttpStream>) -> Next {
+		match self.status {
+			FetchState::Error(ref mut handler) => handler.on_response_writable(encoder),
+			_ => Next::end(),
+		}
+	}
+}
+

dapps/src/handlers/mod.rs

@@ -20,11 +20,14 @@ mod auth;
 mod echo;
 mod content;
 mod redirect;
+mod fetch;
+pub mod client;
 
 pub use self::auth::AuthRequiredHandler;
 pub use self::echo::EchoHandler;
 pub use self::content::ContentHandler;
 pub use self::redirect::Redirection;
+pub use self::fetch::{AppFetcherHandler, DappHandler};
 
 use url::Url;
 use hyper::{server, header, net, uri};

dapps/src/lib.rs

@@ -50,12 +50,15 @@ extern crate hyper;
 extern crate unicase;
 extern crate serde;
 extern crate serde_json;
+extern crate zip;
+extern crate rand;
 extern crate jsonrpc_core;
 extern crate jsonrpc_http_server;
 extern crate parity_dapps;
 extern crate ethcore_rpc;
-extern crate ethcore_util;
+extern crate ethcore_util as util;
 extern crate mime_guess;
+extern crate rustc_serialize;
 
 mod endpoint;
 mod apps;
@@ -121,6 +124,7 @@ impl Server {
 	fn start_http<A: Authorization + 'static>(addr: &SocketAddr, authorization: A, handler: Arc<IoHandler>, dapps_path: String) -> Result<Server, ServerError> {
 		let panic_handler = Arc::new(Mutex::new(None));
 		let authorization = Arc::new(authorization);
+		let apps_fetcher = Arc::new(apps::fetcher::AppFetcher::default());
 		let endpoints = Arc::new(apps::all_endpoints(dapps_path));
 		let special = Arc::new({
 			let mut special = HashMap::new();
@@ -132,8 +136,10 @@ impl Server {
 		let bind_address = format!("{}", addr);
 
 		try!(hyper::Server::http(addr))
-			.handle(move |_| router::Router::new(
+			.handle(move |ctrl| router::Router::new(
+				ctrl,
 				apps::main_page(),
+				apps_fetcher.clone(),
 				endpoints.clone(),
 				special.clone(),
 				authorization.clone(),
@@ -182,3 +188,11 @@ impl From<hyper::error::Error> for ServerError {
 		}
 	}
 }
+
+/// Random filename
+pub fn random_filename() -> String {
+	use ::rand::Rng;
+	let mut rng = ::rand::OsRng::new().unwrap();
+	rng.gen_ascii_chars().take(12).collect()
+}
+

dapps/src/router/host_validation.rs

@@ -16,13 +16,12 @@
 
 
 use DAPPS_DOMAIN;
-use hyper::server;
+use hyper::{server, header};
 use hyper::net::HttpStream;
 
 use jsonrpc_http_server::{is_host_header_valid};
 use handlers::ContentHandler;
 
-
 pub fn is_valid(request: &server::Request<HttpStream>, bind_address: &str, endpoints: Vec<String>) -> bool {
 	let mut endpoints = endpoints.into_iter()
 		.map(|endpoint| format!("{}{}", endpoint, DAPPS_DOMAIN))
@@ -31,7 +30,13 @@ pub fn is_valid(request: &server::Request<HttpStream>, bind_address: &str, endpo
 	endpoints.push(bind_address.replace("127.0.0.1", "localhost").into());
 	endpoints.push(bind_address.into());
 
-	is_host_header_valid(request, &endpoints)
+	let header_valid = is_host_header_valid(request, &endpoints);
+
+	match (header_valid, request.headers().get::<header::Host>()) {
+		(true, _) => true,
+		(_, Some(host)) => host.hostname.ends_with(DAPPS_DOMAIN),
+		_ => false,
+	}
 }
 
 pub fn host_invalid_response() -> Box<server::Handler<HttpStream> + Send> {

dapps/src/router/mod.rs

@@ -24,9 +24,10 @@ use DAPPS_DOMAIN;
 use std::sync::Arc;
 use std::collections::HashMap;
 use url::{Url, Host};
-use hyper::{self, server, Next, Encoder, Decoder};
+use hyper::{self, server, Next, Encoder, Decoder, Control};
 use hyper::net::HttpStream;
 use apps;
+use apps::fetcher::AppFetcher;
 use endpoint::{Endpoint, Endpoints, EndpointPath};
 use handlers::{Redirection, extract_url};
 use self::auth::{Authorization, Authorized};
@@ -41,8 +42,10 @@ pub enum SpecialEndpoint {
 }
 
 pub struct Router<A: Authorization + 'static> {
+	control: Option<Control>,
 	main_page: &'static str,
 	endpoints: Arc<Endpoints>,
+	fetch: Arc<AppFetcher>,
 	special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
 	authorization: Arc<A>,
 	bind_address: String,
@@ -78,6 +81,11 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
 			(Some(ref path), _) if self.endpoints.contains_key(&path.app_id) => {
 				self.endpoints.get(&path.app_id).unwrap().to_handler(path.clone())
 			},
+			// Try to resolve and fetch dapp
+			(Some(ref path), _) if self.fetch.contains(&path.app_id) => {
+				let control = self.control.take().expect("on_request is called only once, thus control is always defined.");
+				self.fetch.to_handler(path.clone(), control)
+			},
 			// Redirection to main page
 			_ if *req.method() == hyper::method::Method::Get => {
 				Redirection::new(self.main_page)
@@ -110,7 +118,9 @@ impl<A: Authorization + 'static> server::Handler<HttpStream> for Router<A> {
 
 impl<A: Authorization> Router<A> {
 	pub fn new(
+		control: Control,
 		main_page: &'static str,
+		app_fetcher: Arc<AppFetcher>,
 		endpoints: Arc<Endpoints>,
 		special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
 		authorization: Arc<A>,
@@ -119,8 +129,10 @@ impl<A: Authorization> Router<A> {
 
 		let handler = special.get(&SpecialEndpoint::Rpc).unwrap().to_handler(EndpointPath::default());
 		Router {
+			control: Some(control),
 			main_page: main_page,
 			endpoints: endpoints,
+			fetch: app_fetcher,
 			special: special,
 			authorization: authorization,
 			bind_address: bind_address,