grassrootseconomics/openethereum
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

LRU cache for dapps (#2006)

Browse Source 
Conflicts:
	dapps/Cargo.toml
	dapps/src/lib.rs
This commit is contained in:
Tomasz Drwięga
2016-08-30 14:04:52 +02:00
committed by Arkadiy Paronyan
parent 6da60afaba
commit 6f321d9849
10 changed files with 214 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
dapps/src/apps/cache.rs Normal file
View File

@@ -0,0 +1,128 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Fetchable Dapps support.
use std::fs;
use std::sync::{Arc};
use std::sync::atomic::{AtomicBool, Ordering};
use linked_hash_map::LinkedHashMap;
use page::LocalPageEndpoint;
pub enum ContentStatus {
Fetching(Arc<AtomicBool>),
Ready(LocalPageEndpoint),
}
#[derive(Default)]
pub struct ContentCache {
cache: LinkedHashMap<String, ContentStatus>,
}
impl ContentCache {
pub fn insert(&mut self, content_id: String, status: ContentStatus) -> Option<ContentStatus> {
self.cache.insert(content_id, status)
}
pub fn remove(&mut self, content_id: &str) -> Option<ContentStatus> {
self.cache.remove(content_id)
}
pub fn get(&mut self, content_id: &str) -> Option<&mut ContentStatus> {
self.cache.get_refresh(content_id)
}
pub fn clear_garbage(&mut self, expected_size: usize) -> Vec<(String, ContentStatus)> {
let mut len = self.cache.len();
if len <= expected_size {
return Vec::new();
}
let mut removed = Vec::with_capacity(len - expected_size);
while len > expected_size {
let entry = self.cache.pop_front().unwrap();
match entry.1 {
ContentStatus::Fetching(ref abort) => {
trace!(target: "dapps", "Aborting {} because of limit.", entry.0);
// Mark as aborted
abort.store(true, Ordering::Relaxed);
},
ContentStatus::Ready(ref endpoint) => {
trace!(target: "dapps", "Removing {} because of limit.", entry.0);
// Remove path
let res = fs::remove_dir_all(&endpoint.path());
if let Err(e) = res {
warn!(target: "dapps", "Unable to remove dapp: {:?}", e);
}
}
}
removed.push(entry);
len -= 1;
}
removed
}
#[cfg(test)]
pub fn len(&self) -> usize {
self.cache.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn only_keys(data: Vec<(String, ContentStatus)>) -> Vec<String> {
data.into_iter().map(|x| x.0).collect()
}
#[test]
fn should_remove_least_recently_used() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));
// when
let res = cache.clear_garbage(2);
// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["a"]);
}
#[test]
fn should_update_lru_if_accessed() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));
// when
cache.get("a");
let res = cache.clear_garbage(2);
// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["b"]);
}
}

51
dapps/src/apps/fetcher.rs
View File

@@ -23,7 +23,7 @@ use std::{fs, env};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool};
use rustc_serialize::hex::FromHex;
use hyper::Control;
@@ -33,20 +33,18 @@ use random_filename;
use util::{Mutex, H256};
use util::sha3::sha3;
use page::LocalPageEndpoint;
use handlers::{ContentHandler, AppFetcherHandler, DappHandler};
use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
use endpoint::{Endpoint, EndpointPath, Handler};
use apps::cache::{ContentCache, ContentStatus};
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
use apps::urlhint::{URLHintContract, URLHint};
enum AppStatus {
Fetching,
Ready(LocalPageEndpoint),
}
const MAX_CACHED_DAPPS: usize = 10;
pub struct AppFetcher<R: URLHint = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}
impl<R: URLHint> Drop for AppFetcher<R> {
@@ -65,17 +63,17 @@ impl<R: URLHint> AppFetcher<R> {
AppFetcher {
dapps_path: dapps_path,
resolver: resolver,
dapps: Arc::new(Mutex::new(HashMap::new())),
dapps: Arc::new(Mutex::new(ContentCache::default())),
}
}
#[cfg(test)]
fn set_status(&self, app_id: &str, status: AppStatus) {
fn set_status(&self, app_id: &str, status: ContentStatus) {
self.dapps.lock().insert(app_id.to_owned(), status);
}
pub fn contains(&self, app_id: &str) -> bool {
let dapps = self.dapps.lock();
let mut dapps = self.dapps.lock();
match dapps.get(app_id) {
// Check if we already have the app
Some(_) => true,
@@ -95,11 +93,11 @@ impl<R: URLHint> AppFetcher<R> {
let status = dapps.get(&app_id);
match status {
// Just server dapp
Some(&AppStatus::Ready(ref endpoint)) => {
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_handler(path))
},
// App is already being fetched
Some(&AppStatus::Fetching) => {
Some(&mut ContentStatus::Fetching(_)) => {
(None, Box::new(ContentHandler::html(
StatusCode::ServiceUnavailable,
format!(
@@ -111,11 +109,13 @@ impl<R: URLHint> AppFetcher<R> {
},
// We need to start fetching app
None => {
// TODO [todr] Keep only last N dapps available!
let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let app = self.resolver.resolve(app_hex).expect("to_handler is called only when `contains` returns true.");
(Some(AppStatus::Fetching), Box::new(AppFetcherHandler::new(
let abort = Arc::new(AtomicBool::new(false));
(Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new(
app,
abort,
control,
path.using_dapps_domains,
DappInstaller {
@@ -129,6 +129,7 @@ impl<R: URLHint> AppFetcher<R> {
};
if let Some(status) = new_status {
dapps.clear_garbage(MAX_CACHED_DAPPS);
dapps.insert(app_id, status);
}
@@ -161,7 +162,7 @@ impl From<zip::result::ZipError> for ValidationError {
struct DappInstaller {
dapp_id: String,
dapps_path: PathBuf,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}
impl DappInstaller {
@@ -196,7 +197,7 @@ impl DappInstaller {
}
}
impl DappHandler for DappInstaller {
impl ContentValidator for DappInstaller {
type Error = ValidationError;
fn validate_and_install(&self, app_path: PathBuf) -> Result<Manifest, ValidationError> {
@@ -262,7 +263,7 @@ impl DappHandler for DappInstaller {
Some(manifest) => {
let path = self.dapp_target_path(manifest);
let app = LocalPageEndpoint::new(path, manifest.clone().into());
dapps.insert(self.dapp_id.clone(), AppStatus::Ready(app));
dapps.insert(self.dapp_id.clone(), ContentStatus::Ready(app));
},
// In case of error
None => {
@@ -274,12 +275,13 @@ impl DappHandler for DappInstaller {
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::{AppFetcher, AppStatus};
use apps::urlhint::{GithubApp, URLHint};
use std::env;
use util::Bytes;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use util::Bytes;
use apps::cache::ContentStatus;
use apps::urlhint::{GithubApp, URLHint};
use super::AppFetcher;
struct FakeResolver;
impl URLHint for FakeResolver {
@@ -291,8 +293,9 @@ mod tests {
#[test]
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = AppFetcher::new(FakeResolver);
let handler = LocalPageEndpoint::new(PathBuf::from("/tmp/test"), EndpointInfo {
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),
@@ -301,8 +304,8 @@ mod tests {
});
// when
fetcher.set_status("test", AppStatus::Ready(handler));
fetcher.set_status("test2", AppStatus::Fetching);
fetcher.set_status("test", ContentStatus::Ready(handler));
fetcher.set_status("test2", ContentStatus::Fetching(Default::default()));
// then
assert_eq!(fetcher.contains("test"), true);

1
dapps/src/apps/mod.rs
View File

@@ -19,6 +19,7 @@ use page::PageEndpoint;
use proxypac::ProxyPac;
use parity_dapps::WebApp;
mod cache;
mod fs;
pub mod urlhint;
pub mod fetcher;