From 845bc52e36003e2ad4553f72b6c547c7a105a89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sun, 20 Nov 2016 17:40:28 +0100 Subject: [PATCH 1/3] Moving contract resolver to separate crate --- Cargo.lock | 13 +++++++++- dapps/Cargo.toml | 4 +-- dapps/src/apps/fetcher.rs | 7 +++--- dapps/src/apps/mod.rs | 1 - dapps/src/lib.rs | 7 +++--- dapps/src/tests/helpers.rs | 2 +- ethcore/hash-fetch/Cargo.toml | 14 +++++++++++ .../hash-fetch/res}/registrar.json | 0 .../hash-fetch/res}/urlhint.json | 0 ethcore/hash-fetch/src/lib.rs | 25 +++++++++++++++++++ .../hash-fetch/src}/urlhint.rs | 4 +-- 11 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 ethcore/hash-fetch/Cargo.toml rename {dapps/src/apps => ethcore/hash-fetch/res}/registrar.json (100%) rename {dapps/src/apps => ethcore/hash-fetch/res}/urlhint.json (100%) create mode 100644 ethcore/hash-fetch/src/lib.rs rename {dapps/src/apps => ethcore/hash-fetch/src}/urlhint.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index 4a209e009..a590d479c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,8 +334,8 @@ version = "1.5.0" dependencies = [ "clippy 0.0.96 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.4.0", + "ethcore-hash-fetch 1.5.0", "ethcore-rpc 1.5.0", "ethcore-util 1.5.0", "fetch 0.1.0", @@ -366,6 +366,17 @@ dependencies = [ "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ethcore-hash-fetch" +version = "1.5.0" +dependencies = [ + "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore-util 1.5.0", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ethcore-io" version = "1.5.0" diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index f6e9d102d..15e537820 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -20,20 +20,20 @@ url = "1.0" rustc-serialize = "0.3" serde = "0.8" serde_json = "0.8" -ethabi = "0.2.2" linked-hash-map = "0.3" parity-dapps-glue = "1.4" mime = "0.2" +mime_guess = "1.6.1" time = "0.1.35" serde_macros = { version = "0.8", optional = true } zip = { version = "0.1", default-features = false } ethcore-devtools = { path = "../devtools" } ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } +ethcore-hash-fetch = { path = "../ethcore/hash-fetch" } fetch = { path = "../util/fetch" } parity-ui = { path = "./ui" } -mime_guess = { version = "1.6.1" } clippy = { version = "0.0.96", optional = true} [build-dependencies] diff --git a/dapps/src/apps/fetcher.rs b/dapps/src/apps/fetcher.rs index 2c1414201..e7f36d144 100644 --- a/dapps/src/apps/fetcher.rs +++ b/dapps/src/apps/fetcher.rs @@ -24,6 +24,7 @@ use std::io::{self, Read, Write}; use std::path::PathBuf; use std::sync::Arc; use rustc_serialize::hex::FromHex; +use hash_fetch::urlhint::{URLHintContract, URLHint, URLHintResult}; use hyper; use hyper::status::StatusCode; @@ -37,7 +38,6 @@ use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator}; use endpoint::{Endpoint, EndpointPath, Handler}; use apps::cache::{ContentCache, ContentStatus}; use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest}; -use apps::urlhint::{URLHintContract, URLHint, URLHintResult}; /// Limit of cached dapps/content const MAX_CACHED_DAPPS: usize = 20; @@ -402,10 +402,11 @@ mod tests { use std::env; use std::sync::Arc; use util::Bytes; + use hash_fetch::urlhint::{URLHint, URLHintResult}; + + use apps::cache::ContentStatus; use endpoint::EndpointInfo; use page::LocalPageEndpoint; - use apps::cache::ContentStatus; - use apps::urlhint::{URLHint, URLHintResult}; use super::ContentFetcher; struct FakeResolver; diff --git a/dapps/src/apps/mod.rs b/dapps/src/apps/mod.rs index 3cb0d8256..4c9270aa5 100644 --- a/dapps/src/apps/mod.rs +++ b/dapps/src/apps/mod.rs @@ -21,7 +21,6 @@ use parity_dapps::WebApp; mod cache; mod fs; -pub mod urlhint; pub mod fetcher; pub mod manifest; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index 2c9fa33d1..7c7ea0a86 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -51,13 +51,13 @@ extern crate serde; extern crate serde_json; extern crate zip; extern crate rand; -extern crate ethabi; extern crate jsonrpc_core; extern crate jsonrpc_http_server; extern crate mime_guess; extern crate rustc_serialize; extern crate ethcore_rpc; extern crate ethcore_util as util; +extern crate ethcore_hash_fetch as hash_fetch; extern crate linked_hash_map; extern crate fetch; extern crate parity_dapps_glue as parity_dapps; @@ -84,12 +84,11 @@ mod url; #[cfg(test)] mod tests; -pub use self::apps::urlhint::ContractClient; - use std::sync::{Arc, Mutex}; use std::net::SocketAddr; use std::collections::HashMap; +use hash_fetch::urlhint::ContractClient; use jsonrpc_core::{IoHandler, IoDelegate}; use router::auth::{Authorization, NoAuth, HttpBasicAuth}; use ethcore_rpc::Extendable; @@ -219,7 +218,7 @@ impl Server { ) -> Result { let panic_handler = Arc::new(Mutex::new(None)); let authorization = Arc::new(authorization); - let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(apps::urlhint::URLHintContract::new(registrar), sync_status, signer_address.clone())); + let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(hash_fetch::urlhint::URLHintContract::new(registrar), sync_status, signer_address.clone())); let endpoints = Arc::new(apps::all_endpoints(dapps_path, signer_address.clone())); let cors_domains = Self::cors_domains(signer_address.clone()); diff --git a/dapps/src/tests/helpers.rs b/dapps/src/tests/helpers.rs index f7c9e8ba6..66bf0f8eb 100644 --- a/dapps/src/tests/helpers.rs +++ b/dapps/src/tests/helpers.rs @@ -22,7 +22,7 @@ use env_logger::LogBuilder; use ServerBuilder; use Server; -use apps::urlhint::ContractClient; +use hash_fetch::urlhint::ContractClient; use util::{Bytes, Address, Mutex, ToPretty}; use devtools::http_client; diff --git a/ethcore/hash-fetch/Cargo.toml b/ethcore/hash-fetch/Cargo.toml new file mode 100644 index 000000000..f1a65bfc8 --- /dev/null +++ b/ethcore/hash-fetch/Cargo.toml @@ -0,0 +1,14 @@ +[package] +description = "Fetching hash-addressed content." +homepage = "https://ethcore.io" +license = "GPL-3.0" +name = "ethcore-hash-fetch" +version = "1.5.0" +authors = ["Ethcore "] + +[dependencies] +log = "0.3" +rustc-serialize = "0.3" +ethabi = "0.2.2" +mime_guess = "1.6.1" +ethcore-util = { path = "../../util" } diff --git a/dapps/src/apps/registrar.json b/ethcore/hash-fetch/res/registrar.json similarity index 100% rename from dapps/src/apps/registrar.json rename to ethcore/hash-fetch/res/registrar.json diff --git a/dapps/src/apps/urlhint.json b/ethcore/hash-fetch/res/urlhint.json similarity index 100% rename from dapps/src/apps/urlhint.json rename to ethcore/hash-fetch/res/urlhint.json diff --git a/ethcore/hash-fetch/src/lib.rs b/ethcore/hash-fetch/src/lib.rs new file mode 100644 index 000000000..ac613a558 --- /dev/null +++ b/ethcore/hash-fetch/src/lib.rs @@ -0,0 +1,25 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#[macro_use] +extern crate log; +extern crate rustc_serialize; +extern crate mime_guess; +extern crate ethabi; +extern crate ethcore_util as util; + +pub mod urlhint; + diff --git a/dapps/src/apps/urlhint.rs b/ethcore/hash-fetch/src/urlhint.rs similarity index 98% rename from dapps/src/apps/urlhint.rs rename to ethcore/hash-fetch/src/urlhint.rs index 27769d07a..b7f905144 100644 --- a/dapps/src/apps/urlhint.rs +++ b/ethcore/hash-fetch/src/urlhint.rs @@ -92,8 +92,8 @@ pub struct URLHintContract { impl URLHintContract { pub fn new(client: Arc) -> Self { - let urlhint = Interface::load(include_bytes!("./urlhint.json")).expect("urlhint.json is valid ABI"); - let registrar = Interface::load(include_bytes!("./registrar.json")).expect("registrar.json is valid ABI"); + let urlhint = Interface::load(include_bytes!("../res/urlhint.json")).expect("urlhint.json is valid ABI"); + let registrar = Interface::load(include_bytes!("../res/registrar.json")).expect("registrar.json is valid ABI"); URLHintContract { urlhint: Contract::new(urlhint), From cc8a9d410b235a17c59ebfc9067c56b53dac56b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sun, 20 Nov 2016 18:30:54 +0100 Subject: [PATCH 2/3] Adding fetch API to the crate --- Cargo.lock | 1 + ethcore/hash-fetch/Cargo.toml | 1 + ethcore/hash-fetch/src/client.rs | 114 ++++++++++++++++++++++++++++++ ethcore/hash-fetch/src/lib.rs | 8 +++ ethcore/hash-fetch/src/urlhint.rs | 48 ++++++++----- util/bigint/src/uint.rs | 1 + util/fetch/src/lib.rs | 2 +- 7 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 ethcore/hash-fetch/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index a590d479c..85aac6e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -372,6 +372,7 @@ version = "1.5.0" dependencies = [ "ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-util 1.5.0", + "fetch 0.1.0", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/hash-fetch/Cargo.toml b/ethcore/hash-fetch/Cargo.toml index f1a65bfc8..4fb724c08 100644 --- a/ethcore/hash-fetch/Cargo.toml +++ b/ethcore/hash-fetch/Cargo.toml @@ -11,4 +11,5 @@ log = "0.3" rustc-serialize = "0.3" ethabi = "0.2.2" mime_guess = "1.6.1" +fetch = { path = "../../util/fetch" } ethcore-util = { path = "../../util" } diff --git a/ethcore/hash-fetch/src/client.rs b/ethcore/hash-fetch/src/client.rs new file mode 100644 index 000000000..f5d19afa5 --- /dev/null +++ b/ethcore/hash-fetch/src/client.rs @@ -0,0 +1,114 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Hash-addressed content resolver & fetcher. + +use std::{io, fs}; +use std::sync::Arc; +use std::path::PathBuf; + +use util::{Mutex, H256, sha3}; +use fetch::{Fetch, FetchError, Client as FetchClient}; + +use urlhint::{ContractClient, URLHintContract, URLHint, URLHintResult}; + +/// API for fetching by hash. +pub trait HashFetch { + /// Fetch hash-addressed content. + /// Parameters: + /// 1. `hash` - content hash + /// 2. `on_done` - callback function invoked when the content is ready (or there was error during fetch) + /// + /// This function may fail immediately when fetch cannot be initialized or content cannot be resolved. + fn fetch(&self, hash: H256, on_done: Box) + Send>) -> Result<(), Error>; +} + +/// Hash-fetching error. +#[derive(Debug)] +pub enum Error { + /// Hash could not be resolved to a valid content address. + NoResolution, + /// Downloaded content hash does not match. + HashMismatch { expected: H256, got: H256 }, + /// IO Error while validating hash. + IO(io::Error), + /// Error during fetch. + Fetch(FetchError), +} + +impl From for Error { + fn from(error: FetchError) -> Self { + Error::Fetch(error) + } +} + +impl From for Error { + fn from(error: io::Error) -> Self { + Error::IO(error) + } +} + +/// Default Hash-fetching client using on-chain contract to resolve hashes to URLs. +pub struct Client { + contract: URLHintContract, + fetch: Mutex, +} + +impl Client { + /// Creates new instance of the `Client` given on-chain contract client. + pub fn new(contract: Arc) -> Self { + Client { + contract: URLHintContract::new(contract), + fetch: Mutex::new(FetchClient::default()), + } + } +} + +impl HashFetch for Client { + fn fetch(&self, hash: H256, on_done: Box) + Send>) -> Result<(), Error> { + debug!(target: "dapps", "Fetching: {:?}", hash); + + let url = try!( + self.contract.resolve(hash.to_vec()).map(|content| match content { + URLHintResult::Dapp(dapp) => { + dapp.url() + }, + URLHintResult::Content(content) => { + content.url + }, + }).ok_or_else(|| Error::NoResolution) + ); + + debug!(target: "dapps", "Resolved {:?} to {:?}. Fetching...", hash, url); + + self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| { + fn validate_hash(hash: H256, result: Result) -> Result { + let path = try!(result); + let mut file_reader = io::BufReader::new(try!(fs::File::open(&path))); + let content_hash = try!(sha3(&mut file_reader)); + + if content_hash != hash { + Err(Error::HashMismatch{ got: content_hash, expected: hash }) + } else { + Ok(path) + } + } + + debug!(target: "dapps", "Content fetched, validating hash ({:?})", hash); + on_done(validate_hash(hash, result)) + })).map_err(Into::into) + } +} diff --git a/ethcore/hash-fetch/src/lib.rs b/ethcore/hash-fetch/src/lib.rs index ac613a558..ffb74b260 100644 --- a/ethcore/hash-fetch/src/lib.rs +++ b/ethcore/hash-fetch/src/lib.rs @@ -14,12 +14,20 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +//! Hash-addressed content resolver & fetcher. + +#![warn(missing_docs)] + #[macro_use] extern crate log; extern crate rustc_serialize; extern crate mime_guess; extern crate ethabi; extern crate ethcore_util as util; +extern crate fetch; + +mod client; pub mod urlhint; +pub use client::{HashFetch, Client}; diff --git a/ethcore/hash-fetch/src/urlhint.rs b/ethcore/hash-fetch/src/urlhint.rs index b7f905144..9cbd13b1e 100644 --- a/ethcore/hash-fetch/src/urlhint.rs +++ b/ethcore/hash-fetch/src/urlhint.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +//! URLHint Contract + use std::fmt; use std::sync::Arc; use rustc_serialize::hex::ToHex; @@ -24,15 +26,30 @@ use util::{Address, Bytes, Hashable}; const COMMIT_LEN: usize = 20; +/// RAW Contract interface. +/// Should execute transaction using current blockchain state. +pub trait ContractClient: Send + Sync { + /// Get registrar address + fn registrar(&self) -> Result; + /// Call Contract + fn call(&self, address: Address, data: Bytes) -> Result; +} + +/// Github-hosted dapp. #[derive(Debug, PartialEq)] pub struct GithubApp { + /// Github Account pub account: String, + /// Github Repository pub repo: String, + /// Commit on Github pub commit: [u8;COMMIT_LEN], + /// Dapp owner address pub owner: Address, } impl GithubApp { + /// Returns URL of this Github-hosted dapp package. pub fn url(&self) -> String { // Since https fetcher doesn't support redirections we use direct link // format!("https://github.com/{}/{}/archive/{}.zip", self.account, self.repo, self.commit.to_hex()) @@ -53,22 +70,17 @@ impl GithubApp { } } +/// Hash-Addressed Content #[derive(Debug, PartialEq)] pub struct Content { + /// URL of the content pub url: String, + /// MIME type of the content pub mime: String, + /// Content owner address pub owner: Address, } -/// RAW Contract interface. -/// Should execute transaction using current blockchain state. -pub trait ContractClient: Send + Sync { - /// Get registrar address - fn registrar(&self) -> Result; - /// Call Contract - fn call(&self, address: Address, data: Bytes) -> Result; -} - /// Result of resolving id to URL #[derive(Debug, PartialEq)] pub enum URLHintResult { @@ -84,6 +96,7 @@ pub trait URLHint { fn resolve(&self, id: Bytes) -> Option; } +/// `URLHintContract` API pub struct URLHintContract { urlhint: Contract, registrar: Contract, @@ -91,6 +104,7 @@ pub struct URLHintContract { } impl URLHintContract { + /// Creates new `URLHintContract` pub fn new(client: Arc) -> Self { let urlhint = Interface::load(include_bytes!("../res/urlhint.json")).expect("urlhint.json is valid ABI"); let registrar = Interface::load(include_bytes!("../res/registrar.json")).expect("registrar.json is valid ABI"); @@ -244,11 +258,6 @@ fn guess_mime_type(url: &str) -> Option { }) } -#[cfg(test)] -pub fn test_guess_mime_type(url: &str) -> Option { - guess_mime_type(url) -} - fn as_string(e: T) -> String { format!("{:?}", e) } @@ -260,6 +269,7 @@ mod tests { use rustc_serialize::hex::FromHex; use super::*; + use super::guess_mime_type; use util::{Bytes, Address, Mutex, ToPretty}; struct FakeRegistrar { @@ -390,10 +400,10 @@ mod tests { let url5 = "https://ethcore.io/parity.png"; - assert_eq!(test_guess_mime_type(url1), None); - assert_eq!(test_guess_mime_type(url2), Some("image/png".into())); - assert_eq!(test_guess_mime_type(url3), Some("image/png".into())); - assert_eq!(test_guess_mime_type(url4), Some("image/jpeg".into())); - assert_eq!(test_guess_mime_type(url5), Some("image/png".into())); + assert_eq!(guess_mime_type(url1), None); + assert_eq!(guess_mime_type(url2), Some("image/png".into())); + assert_eq!(guess_mime_type(url3), Some("image/png".into())); + assert_eq!(guess_mime_type(url4), Some("image/jpeg".into())); + assert_eq!(guess_mime_type(url5), Some("image/png".into())); } } diff --git a/util/bigint/src/uint.rs b/util/bigint/src/uint.rs index dab00537e..f4dd91140 100644 --- a/util/bigint/src/uint.rs +++ b/util/bigint/src/uint.rs @@ -683,6 +683,7 @@ macro_rules! construct_uint { bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; } } + #[inline] fn exp10(n: usize) -> Self { match n { diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index 7ab38604b..8ec9e0ddd 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -26,4 +26,4 @@ extern crate rand; pub mod client; pub mod fetch_file; -pub use self::client::{Client, Fetch, FetchError, FetchResult}; \ No newline at end of file +pub use self::client::{Client, Fetch, FetchError, FetchResult}; From 94328e97845aef1ced5078ebd9ab80b156fb8643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sun, 20 Nov 2016 22:40:23 +0100 Subject: [PATCH 3/3] Fixing main --- Cargo.lock | 1 + Cargo.toml | 1 + parity/dapps.rs | 2 +- parity/main.rs | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 85aac6e54..609d02dab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,7 @@ dependencies = [ "ethcore 1.5.0", "ethcore-dapps 1.5.0", "ethcore-devtools 1.4.0", + "ethcore-hash-fetch 1.5.0", "ethcore-io 1.5.0", "ethcore-ipc 1.4.0", "ethcore-ipc-codegen 1.4.0", diff --git a/Cargo.toml b/Cargo.toml index fe72d67ec..a8d7ba794 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ ethcore-ipc-nano = { path = "ipc/nano" } ethcore-ipc = { path = "ipc/rpc" } ethcore-ipc-hypervisor = { path = "ipc/hypervisor" } ethcore-logger = { path = "logger" } +ethcore-hash-fetch = { path = "ethcore/hash-fetch" } rlp = { path = "util/rlp" } ethcore-stratum = { path = "stratum" } ethcore-dapps = { path = "dapps", optional = true } diff --git a/parity/dapps.rs b/parity/dapps.rs index 80f2f7060..16ae4dd98 100644 --- a/parity/dapps.rs +++ b/parity/dapps.rs @@ -110,7 +110,7 @@ mod server { use rpc_apis; use ethcore_rpc::is_major_importing; - use ethcore_dapps::ContractClient; + use hash_fetch::urlhint::ContractClient; pub use ethcore_dapps::Server as WebappServer; diff --git a/parity/main.rs b/parity/main.rs index 0fc88d8e7..274d29de2 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -42,6 +42,7 @@ extern crate ethcore_ipc_nano as nanoipc; extern crate serde; extern crate serde_json; extern crate rlp; +extern crate ethcore_hash_fetch as hash_fetch; extern crate json_ipc_server as jsonipc;