Dapp refresh (#5752)

* RwLock

* getting there

* argh

* parking_lot

* rpc

* wax on wax off

* almost there

* remove lock

* write over read

* works

* linting

* small updates

* dissapearing act

* router update

* complete

* one m

* grumbles1

* grumbles part II

* parking_lot->util

* missed test case

* fied package-lock.json

* small fixes

* 404 tests failing

* cleanup

* cleanup 2

* updates and the likes

* play

* simplify filter

* f-ing bugs

* read->write

* Address own grumbles.

* Fix test.

dapps/src/apps/mod.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::collections::BTreeMap;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -30,8 +29,8 @@ use {WebProxyTokens, ParentFrameSettings};
 
 mod app;
 mod cache;
-mod fs;
 mod ui;
+pub mod fs;
 pub mod fetcher;
 pub mod manifest;
 
@@ -64,9 +63,10 @@ pub fn all_endpoints<F: Fetch>(
 	web_proxy_tokens: Arc<WebProxyTokens>,
 	remote: Remote,
 	fetch: F,
-) -> Endpoints {
+) -> (Vec<String>, Endpoints) {
 	// fetch fs dapps at first to avoid overwriting builtins
-	let mut pages = fs::local_endpoints(dapps_path, embeddable.clone());
+	let mut pages = fs::local_endpoints(dapps_path.clone(), embeddable.clone());
+	let local_endpoints: Vec<String> = pages.keys().cloned().collect();
 	for path in extra_dapps {
 		if let Some((id, endpoint)) = fs::local_endpoint(path.clone(), embeddable.clone()) {
 			pages.insert(id, endpoint);
@@ -80,10 +80,10 @@ pub fn all_endpoints<F: Fetch>(
 	pages.insert("proxy".into(), ProxyPac::boxed(embeddable.clone(), dapps_domain.to_owned()));
 	pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), remote.clone(), fetch.clone()));
 
-	Arc::new(pages)
+	(local_endpoints, pages)
 }
 
-fn insert<T : WebApp + Default + 'static>(pages: &mut BTreeMap<String, Box<Endpoint>>, id: &str, embed_at: Embeddable) {
+fn insert<T : WebApp + Default + 'static>(pages: &mut Endpoints, id: &str, embed_at: Embeddable) {
 	pages.insert(id.to_owned(), Box::new(match embed_at {
 		Embeddable::Yes(address) => PageEndpoint::new_safe_to_embed(T::default(), address),
 		Embeddable::No => PageEndpoint::new(T::default()),

dapps/src/endpoint.rs

@@ -16,7 +16,6 @@
 
 //! URL Endpoint traits
 
-use std::sync::Arc;
 use std::collections::BTreeMap;
 
 use hyper::{self, server, net};
@@ -39,7 +38,7 @@ pub struct EndpointInfo {
 	pub icon_url: String,
 }
 
-pub type Endpoints = Arc<BTreeMap<String, Box<Endpoint>>>;
+pub type Endpoints = BTreeMap<String, Box<Endpoint>>;
 pub type Handler = server::Handler<net::HttpStream> + Send;
 
 pub trait Endpoint : Send + Sync {

dapps/src/lib.rs

@@ -69,9 +69,11 @@ mod web;
 #[cfg(test)]
 mod tests;
 
+use std::collections::HashMap;
+use std::mem;
 use std::path::PathBuf;
 use std::sync::Arc;
-use std::collections::HashMap;
+use util::RwLock;
 
 use jsonrpc_http_server::{self as http, hyper, Origin};
 
@@ -101,31 +103,54 @@ impl<F> WebProxyTokens for F where F: Fn(String) -> Option<Origin> + Send + Sync
 }
 
 /// Current supported endpoints.
+#[derive(Default, Clone)]
 pub struct Endpoints {
-	endpoints: endpoint::Endpoints,
+	local_endpoints: Arc<RwLock<Vec<String>>>,
+	endpoints: Arc<RwLock<endpoint::Endpoints>>,
+	dapps_path: PathBuf,
+	embeddable: Option<ParentFrameSettings>,
 }
 
 impl Endpoints {
 	/// Returns a current list of app endpoints.
 	pub fn list(&self) -> Vec<apps::App> {
-		self.endpoints.iter().filter_map(|(ref k, ref e)| {
+		self.endpoints.read().iter().filter_map(|(ref k, ref e)| {
 			e.info().map(|ref info| apps::App::from_info(k, info))
 		}).collect()
 	}
+
+	/// Check for any changes in the local dapps folder and update.
+	pub fn refresh_local_dapps(&self) {
+		let new_local = apps::fs::local_endpoints(&self.dapps_path, self.embeddable.clone());
+		let old_local = mem::replace(&mut *self.local_endpoints.write(), new_local.keys().cloned().collect());
+		let (_, to_remove): (_, Vec<_>) = old_local
+			.into_iter()
+			.partition(|k| new_local.contains_key(&k.clone()));
+
+		let mut endpoints = self.endpoints.write();
+		// remove the dead dapps
+		for k in to_remove {
+			endpoints.remove(&k);
+		}
+		// new dapps to be added
+		for (k, v) in new_local {
+			if !endpoints.contains_key(&k) {
+				endpoints.insert(k, v);
+			}
+		}
+	}
 }
 
 /// Dapps server as `jsonrpc-http-server` request middleware.
 pub struct Middleware {
+	endpoints: Endpoints,
 	router: router::Router,
-	endpoints: endpoint::Endpoints,
 }
 
 impl Middleware {
 	/// Get local endpoints handle.
-	pub fn endpoints(&self) -> Endpoints {
-		Endpoints {
-			endpoints: self.endpoints.clone(),
-		}
+	pub fn endpoints(&self) -> &Endpoints {
+		&self.endpoints
 	}
 
 	/// Creates new middleware for UI server.
@@ -164,8 +189,8 @@ impl Middleware {
 		);
 
 		Middleware {
-			router: router,
 			endpoints: Default::default(),
+			router: router,
 		}
 	}
 
@@ -191,8 +216,8 @@ impl Middleware {
 			remote.clone(),
 			fetch.clone(),
 		).embeddable_on(embeddable.clone()).allow_dapps(true));
-		let endpoints = apps::all_endpoints(
-			dapps_path,
+		let (local_endpoints, endpoints) = apps::all_endpoints(
+			dapps_path.clone(),
 			extra_dapps,
 			dapps_domain,
 			embeddable.clone(),
@@ -200,6 +225,12 @@ impl Middleware {
 			remote.clone(),
 			fetch.clone(),
 		);
+		let endpoints = Endpoints {
+			endpoints: Arc::new(RwLock::new(endpoints)),
+			dapps_path,
+			local_endpoints: Arc::new(RwLock::new(local_endpoints)),
+			embeddable: embeddable.clone(),
+		};
 
 		let special = {
 			let mut special = special_endpoints(
@@ -225,8 +256,8 @@ impl Middleware {
 		);
 
 		Middleware {
-			router: router,
-			endpoints: endpoints,
+			endpoints,
+			router,
 		}
 	}
 }

dapps/src/router.rs

@@ -28,7 +28,8 @@ use jsonrpc_http_server as http;
 
 use apps;
 use apps::fetcher::Fetcher;
-use endpoint::{Endpoint, Endpoints, EndpointPath, Handler};
+use endpoint::{Endpoint, EndpointPath, Handler};
+use Endpoints;
 use handlers;
 use Embeddable;
 
@@ -50,26 +51,27 @@ pub struct Router {
 	dapps_domain: String,
 }
 
-impl http::RequestMiddleware for Router {
-	fn on_request(&self, req: &server::Request<HttpStream>, control: &Control) -> http::RequestMiddlewareAction {
+impl Router {
+	fn resolve_request(&self, req: &server::Request<HttpStream>, control: Control, refresh_dapps: bool) -> (bool, Option<Box<Handler>>) {
 		// Choose proper handler depending on path / domain
 		let url = handlers::extract_url(req);
 		let endpoint = extract_endpoint(&url, &self.dapps_domain);
 		let referer = extract_referer_endpoint(req, &self.dapps_domain);
 		let is_utils = endpoint.1 == SpecialEndpoint::Utils;
-		let is_origin_set = req.headers().get::<header::Origin>().is_some();
 		let is_get_request = *req.method() == hyper::Method::Get;
 		let is_head_request = *req.method() == hyper::Method::Head;
+		let has_dapp = |dapp: &str| self.endpoints
+			.as_ref()
+			.map_or(false, |endpoints| endpoints.endpoints.read().contains_key(dapp));
 
 		trace!(target: "dapps", "Routing request to {:?}. Details: {:?}", url, req);
-
-		let control = control.clone();
 		debug!(target: "dapps", "Handling endpoint request: {:?}", endpoint);
-		let handler: Option<Box<Handler>> = match (endpoint.0, endpoint.1, referer) {
+
+		(is_utils, match (endpoint.0, endpoint.1, referer) {
 			// Handle invalid web requests that we can recover from
 			(ref path, SpecialEndpoint::None, Some((ref referer, ref referer_url)))
 				if referer.app_id == apps::WEB_PATH
-					&& self.endpoints.as_ref().map(|ep| ep.contains_key(apps::WEB_PATH)).unwrap_or(false)
+					&& has_dapp(apps::WEB_PATH)
 					&& !is_web_endpoint(path)
 				=>
 			{
@@ -88,11 +90,13 @@ impl http::RequestMiddleware for Router {
 					.map(|special| special.to_async_handler(path.clone().unwrap_or_default(), control))
 			},
 			// Then delegate to dapp
-			(Some(ref path), _, _) if self.endpoints.as_ref().map(|ep| ep.contains_key(&path.app_id)).unwrap_or(false) => {
+			(Some(ref path), _, _) if has_dapp(&path.app_id) => {
 				trace!(target: "dapps", "Resolving to local/builtin dapp.");
 				Some(self.endpoints
 					.as_ref()
 					.expect("endpoints known to be set; qed")
+					.endpoints
+					.read()
 					.get(&path.app_id)
 					.expect("endpoints known to contain key; qed")
 					.to_async_handler(path.clone(), control))
@@ -110,13 +114,19 @@ impl http::RequestMiddleware for Router {
 				=>
 			{
 				trace!(target: "dapps", "Resolving to 404.");
-				Some(Box::new(handlers::ContentHandler::error(
-					hyper::StatusCode::NotFound,
-					"404 Not Found",
-					"Requested content was not found.",
-					None,
-					self.embeddable_on.clone(),
-				)))
+				if refresh_dapps {
+					debug!(target: "dapps", "Refreshing dapps and re-trying.");
+					self.endpoints.as_ref().map(|endpoints| endpoints.refresh_local_dapps());
+					return self.resolve_request(req, control, false)
+				} else {
+					Some(Box::new(handlers::ContentHandler::error(
+						hyper::StatusCode::NotFound,
+						"404 Not Found",
+						"Requested content was not found.",
+						None,
+						self.embeddable_on.clone(),
+					)))
+				}
 			},
 			// Any other GET|HEAD requests to home page.
 			_ if (is_get_request || is_head_request) && self.special.contains_key(&SpecialEndpoint::Home) => {
@@ -130,8 +140,15 @@ impl http::RequestMiddleware for Router {
 				trace!(target: "dapps", "Resolving to RPC call.");
 				None
 			}
-		};
+		})
+	}
+}
 
+impl http::RequestMiddleware for Router {
+	fn on_request(&self, req: &server::Request<HttpStream>, control: &Control) -> http::RequestMiddlewareAction {
+		let control = control.clone();
+		let is_origin_set = req.headers().get::<header::Origin>().is_some();
+		let (is_utils, handler) = self.resolve_request(req, control, self.endpoints.is_some());
 		match handler {
 			Some(handler) => http::RequestMiddlewareAction::Respond {
 				should_validate_hosts: !is_utils,

dapps/src/tests/fetch.rs

@@ -39,7 +39,7 @@ fn should_resolve_dapp() {
 
 	// then
 	response.assert_status("HTTP/1.1 404 Not Found");
-	assert_eq!(registrar.calls.lock().len(), 2);
+	assert_eq!(registrar.calls.lock().len(), 4);
 	assert_security_headers_for_embed(&response.headers);
 }
 

dapps/src/tests/redirection.rs

@@ -204,4 +204,3 @@ fn should_serve_utils() {
 	assert_eq!(response.body.contains("function(){"), true);
 	assert_security_headers(&response.headers);
 }
-

dapps/src/web.rs

@@ -241,5 +241,3 @@ impl<F: Fetch> server::Handler<net::HttpStream> for WebHandler<F> {
 		}
 	}
 }
-
-

js/package-lock.json

@@ -7722,7 +7722,7 @@
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=",
       "requires": {
         "brace-expansion": "1.1.8"
       }
@@ -10081,7 +10081,7 @@
     "react-qr-reader": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/react-qr-reader/-/react-qr-reader-1.1.3.tgz",
-      "integrity": "sha512-ruBF8KaSwUW9nbzjO4rA7/HOCGYZuNUz9od7uBRy8SRBi24nwxWWmwa2z8R6vPGDRglA0y2Qk1aVBuC1olTnHw==",
+      "integrity": "sha1-dDmnZvyZPLj17u/HLCnblh1AswI=",
       "requires": {
         "jsqr": "git+https://github.com/JodusNodus/jsQR.git#5ba1acefa1cbb9b2bc92b49f503f2674e2ec212b",
         "prop-types": "15.5.10",

js/src/api/rpc/parity/parity.js

@@ -95,6 +95,11 @@ export default class Parity {
       .execute('parity_dappsList');
   }
 
+  dappsRefresh () {
+    return this._transport
+      .execute('parity_dappsRefresh');
+  }
+
   dappsUrl () {
     return this._transport
       .execute('parity_dappsUrl');

js/src/jsonrpc/interfaces/parity.js

@@ -164,6 +164,17 @@ export default {
     }
   },
 
+  dappsRefresh: {
+    subdoc: SUBDOC_SET,
+    desc: 'Returns a boolean value upon success and error upon failure',
+    params: [],
+    returns: {
+      type: Boolean,
+      desc: 'True for success. error details for failure',
+      example: true
+    }
+  },
+
   dappsUrl: {
     section: SECTION_NODE,
     desc: 'Returns the hostname and the port of dapps/rpc server, error if not enabled.',

js/src/views/Dapps/dapps.js

@@ -24,7 +24,7 @@ import { connect } from 'react-redux';
 import { DappPermissions, DappsVisible } from '~/modals';
 import PermissionStore from '~/modals/DappPermissions/store';
 import { Actionbar, Button, DappCard, Page, SectionList } from '~/ui';
-import { LockedIcon, VisibleIcon } from '~/ui/Icons';
+import { LockedIcon, RefreshIcon, VisibleIcon } from '~/ui/Icons';
 
 import DappsStore from './dappsStore';
 
@@ -90,6 +90,17 @@ class Dapps extends Component {
             />
           }
           buttons={ [
+            <Button
+              icon={ <RefreshIcon /> }
+              key='refresh'
+              label={
+                <FormattedMessage
+                  id='dapps.button.dapp.refresh'
+                  defaultMessage='refresh'
+                />
+              }
+              onClick={ this.store.refreshDapps }
+            />,
             <Button
               icon={ <VisibleIcon /> }
               key='edit'

js/src/views/Dapps/dappsStore.js

@@ -89,7 +89,7 @@ export default class DappsStore extends EventEmitter {
     return Promise
       .all([
         this.fetchBuiltinApps().then((apps) => this.addApps(apps)),
-        this.fetchLocalApps().then((apps) => this.addApps(apps))
+        this.fetchLocalApps().then((apps) => this.addApps(apps, true))
       ]);
   }
 
@@ -227,6 +227,20 @@ export default class DappsStore extends EventEmitter {
     return this.visibleApps.filter((app) => app.type === 'network');
   }
 
+  @action refreshDapps = () => {
+    const self = this;
+
+    self._api.parity.dappsRefresh()
+      .then((res) => {
+        if (res === true) {
+          self.loadAllApps();
+        }
+      })
+      .catch((err) => {
+        console.log(err);
+      });
+  }
+
   @action openModal = () => {
     this.modalOpen = true;
   }
@@ -266,7 +280,7 @@ export default class DappsStore extends EventEmitter {
     this.displayApps = Object.assign({}, this.displayApps, displayApps);
   };
 
-  @action addApps = (_apps = []) => {
+  @action addApps = (_apps = [], _local = false) => {
     transaction(() => {
       const apps = _apps.filter((app) => app);
 
@@ -277,6 +291,7 @@ export default class DappsStore extends EventEmitter {
 
       this.apps = this.apps
         .filter((app) => !app.id || !newAppsIds.includes(app.id))
+        .filter((app) => !(app.type === 'local' && _local && apps.indexOf(app) === -1))
         .concat(apps || [])
         .sort((a, b) => a.name.localeCompare(b.name));
 

parity/dapps.rs

@@ -291,7 +291,7 @@ mod server {
 
 	pub fn service(middleware: &Option<Middleware>) -> Option<Arc<rpc_apis::DappsService>> {
 		middleware.as_ref().map(|m| Arc::new(DappsServiceWrapper {
-			endpoints: m.endpoints()
+			endpoints: m.endpoints().clone(),
 		}) as Arc<rpc_apis::DappsService>)
 	}
 
@@ -313,5 +313,10 @@ mod server {
 				})
 				.collect()
 		}
+
+		fn refresh_local_dapps(&self) -> bool {
+			self.endpoints.refresh_local_dapps();
+			true
+		}
 	}
 }

rpc/src/v1/helpers/dapps.rs

@@ -22,12 +22,6 @@ use v1::types::LocalDapp;
 pub trait DappsService: Send + Sync + 'static {
 	/// List available local dapps.
 	fn list_dapps(&self) -> Vec<LocalDapp>;
-}
-
-impl<F> DappsService for F where
-	F: Fn() -> Vec<LocalDapp> + Send + Sync + 'static
-{
-	fn list_dapps(&self) -> Vec<LocalDapp> {
-		(*self)()
-	}
+	/// Refresh local dapps list
+	fn refresh_local_dapps(&self) -> bool;
 }

rpc/src/v1/impls/light/parity_set.rs

@@ -135,6 +135,10 @@ impl<F: Fetch> ParitySet for ParitySetClient<F> {
 		}))
 	}
 
+	fn dapps_refresh(&self) -> Result<bool, Error> {
+		self.dapps.as_ref().map(|dapps| dapps.refresh_local_dapps()).ok_or_else(errors::dapps_disabled)
+	}
+
 	fn dapps_list(&self) -> Result<Vec<LocalDapp>, Error> {
 		self.dapps.as_ref().map(|dapps| dapps.list_dapps()).ok_or_else(errors::dapps_disabled)
 	}

rpc/src/v1/impls/parity_set.rs

@@ -176,6 +176,10 @@ impl<C, M, U, F> ParitySet for ParitySetClient<C, M, U, F> where
 		}))
 	}
 
+	fn dapps_refresh(&self) -> Result<bool, Error> {
+		self.dapps.as_ref().map(|dapps| dapps.refresh_local_dapps()).ok_or_else(errors::dapps_disabled)
+	}
+
 	fn dapps_list(&self) -> Result<Vec<LocalDapp>, Error> {
 		self.dapps.as_ref().map(|dapps| dapps.list_dapps()).ok_or_else(errors::dapps_disabled)
 	}

rpc/src/v1/tests/helpers/dapps.rs

@@ -34,4 +34,8 @@ impl DappsService for TestDappsService {
 			icon_url: "title.png".into(),
 		}]
 	}
+
+	fn refresh_local_dapps(&self) -> bool {
+		true
+	}
 }

rpc/src/v1/traits/parity_set.rs

@@ -96,6 +96,10 @@ build_rpc_trait! {
 		#[rpc(async, name = "parity_hashContent")]
 		fn hash_content(&self, String) -> BoxFuture<H256, Error>;
 
+		/// Returns true if refresh successful, error if unsuccessful or server is disabled.
+		#[rpc(name = "parity_dappsRefresh")]
+		fn dapps_refresh(&self) -> Result<bool, Error>;
+
 		/// Returns a list of local dapps
 		#[rpc(name = "parity_dappsList")]
 		fn dapps_list(&self) -> Result<Vec<LocalDapp>, Error>;