Global Fetch Service (#3915)

* Dapps web

Conflicts:
	dapps/src/apps/fetcher.rs
	dapps/src/handlers/fetch.rs

* Rewriting fetch

* Parity-wide fetch service

* Obey the limits and support cancellation.

* Removing temporary files.

* Actually use Fetch for dapps

* Re-implementing file fetching to avoid temporary files.

* Serde to 0.8.19

* Fixing content & dapps fetch
This commit is contained in:
Tomasz Drwięga 2016-12-22 18:26:39 +01:00 committed by Gav Wood
parent 1645419bc8
commit f0387c33c6
45 changed files with 1677 additions and 6617 deletions

346
Cargo.lock generated
View File

@ -26,13 +26,14 @@ dependencies = [
"ethcore-util 1.5.0",
"ethsync 1.5.0",
"fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
"isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-hash-fetch 1.5.0",
"parity-reactor 0.1.0",
"parity-rpc-client 1.4.0",
"parity-updater 1.5.0",
"regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)",
@ -42,13 +43,22 @@ dependencies = [
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "advapi32-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "aho-corasick"
version = "0.5.1"
@ -98,11 +108,6 @@ dependencies = [
"syntex_syntax 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "base64"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bit-set"
version = "0.4.0"
@ -126,6 +131,11 @@ name = "bitflags"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "blastfig"
version = "0.3.3"
@ -200,9 +210,35 @@ dependencies = [
]
[[package]]
name = "crossbeam"
version = "0.2.9"
name = "core-foundation"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"core-foundation-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "core-foundation-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "crossbeam"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "crypt32-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ctrlc"
@ -281,7 +317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"docopt 0.6.80 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -305,7 +341,7 @@ dependencies = [
"bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"clippy 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"env_logger 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"ethash 1.5.0",
@ -364,6 +400,7 @@ dependencies = [
"ethcore-rpc 1.5.0",
"ethcore-util 1.5.0",
"fetch 0.1.0",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)",
"jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
"jsonrpc-http-server 6.1.1 (git+https://github.com/ethcore/jsonrpc.git)",
@ -373,10 +410,11 @@ dependencies = [
"mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-dapps-glue 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-hash-fetch 1.5.0",
"parity-reactor 0.1.0",
"parity-ui 1.5.0",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
@ -396,7 +434,7 @@ dependencies = [
name = "ethcore-io"
version = "1.5.0"
dependencies = [
"crossbeam 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.1 (git+https://github.com/ethcore/mio)",
"parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -531,16 +569,19 @@ dependencies = [
"ethstore 0.1.0",
"ethsync 1.5.0",
"fetch 0.1.0",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
"jsonrpc-http-server 6.1.1 (git+https://github.com/ethcore/jsonrpc.git)",
"jsonrpc-ipc-server 0.2.4 (git+https://github.com/ethcore/jsonrpc.git)",
"jsonrpc-macros 0.1.0 (git+https://github.com/ethcore/jsonrpc.git)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-reactor 0.1.0",
"parity-updater 1.5.0",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rlp 0.1.0",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
@ -637,7 +678,7 @@ version = "0.1.0"
dependencies = [
"ethcore-util 1.5.0",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -670,7 +711,7 @@ dependencies = [
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
@ -721,10 +762,11 @@ dependencies = [
name = "fetch"
version = "0.1.0"
dependencies = [
"https-fetch 0.1.0",
"hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -744,6 +786,16 @@ dependencies = [
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "futures-cpupool"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gcc"
version = "0.3.35"
@ -752,6 +804,15 @@ dependencies = [
"rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "gdi32-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "glob"
version = "0.2.11"
@ -783,19 +844,9 @@ name = "httparse"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "https-fetch"
version = "0.1.0"
dependencies = [
"ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)",
"rustls 0.1.2 (git+https://github.com/ctz/rustls)",
]
[[package]]
name = "hyper"
version = "0.9.10"
version = "0.9.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -803,7 +854,7 @@ dependencies = [
"language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
@ -847,7 +898,7 @@ name = "igd"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"hyper 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)",
"xml-rs 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -892,7 +943,7 @@ source = "git+https://github.com/ethcore/jsonrpc.git#33262d626a294a00c20435dec33
dependencies = [
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -929,7 +980,7 @@ version = "0.1.0"
source = "git+https://github.com/ethcore/jsonrpc.git#33262d626a294a00c20435dec331058ba65e224a"
dependencies = [
"jsonrpc-core 4.0.0 (git+https://github.com/ethcore/jsonrpc.git)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1103,6 +1154,22 @@ dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "mio"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazycell 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"miow 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"net2 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)",
"nix 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "miow"
version = "0.1.3"
@ -1142,6 +1209,18 @@ dependencies = [
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "native-tls"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"openssl 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
"schannel 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"security-framework 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"security-framework-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "net2"
version = "0.2.23"
@ -1276,6 +1355,14 @@ dependencies = [
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num_cpus"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "number_prefix"
version = "0.2.5"
@ -1298,6 +1385,28 @@ dependencies = [
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "openssl"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "openssl-sys"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "owning_ref"
version = "0.2.2"
@ -1324,11 +1433,23 @@ dependencies = [
"ethabi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"ethcore-util 1.5.0",
"fetch 0.1.0",
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-reactor 0.1.0",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "parity-reactor"
version = "0.1.0"
dependencies = [
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-core 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "parity-rpc-client"
version = "1.4.0"
@ -1342,7 +1463,7 @@ dependencies = [
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1386,6 +1507,7 @@ dependencies = [
"ipc-common-types 1.5.0",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-hash-fetch 1.5.0",
"parity-reactor 0.1.0",
"target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1444,6 +1566,11 @@ dependencies = [
"unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "pkg-config"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "podio"
version = "0.1.5"
@ -1574,12 +1701,17 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "ring"
version = "0.4.3"
name = "reqwest"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1682,16 +1814,51 @@ dependencies = [
]
[[package]]
name = "rustls"
version = "0.1.2"
source = "git+https://github.com/ctz/rustls#3d2db624997004b7b18ba4463d6081f37598b2f5"
name = "schannel"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crypt32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"secur32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "scoped-tls"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "secur32-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "security-framework"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"core-foundation 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"core-foundation-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"security-framework-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "security-framework-sys"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"core-foundation-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1726,7 +1893,7 @@ dependencies = [
[[package]]
name = "serde"
version = "0.8.4"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -1759,7 +1926,16 @@ dependencies = [
"dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_urlencoded"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -1978,6 +2154,18 @@ name = "tiny-keccak"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tokio-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "toml"
version = "0.1.28"
@ -2038,11 +2226,6 @@ name = "unicode-xid"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "untrusted"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "url"
version = "1.2.0"
@ -2052,6 +2235,15 @@ dependencies = [
"matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "user32-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "using_queue"
version = "0.1.0"
@ -2084,17 +2276,6 @@ name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "webpki"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi"
version = "0.2.8"
@ -2162,17 +2343,18 @@ dependencies = [
]
[metadata]
"checksum advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a"
"checksum aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67077478f0a03952bed2e6786338d400d40c25e9836e08ad50af96607317fd03"
"checksum ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1f46cd5b1d660c938e3f92dfe7a73d832b3281479363dd0cd9c1c2fbf60f7962"
"checksum app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7d1c0d48a81bbb13043847f957971f4d87c81542d80ece5e84ba3cba4058fd4"
"checksum arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "16e3bdb2f54b3ace0285975d59a97cf8ed3855294b2b6bc651fcf22a9c352975"
"checksum aster 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07d344974f0a155f091948aa389fb1b912d3a58414fbdb9c8d446d193ee3496a"
"checksum aster 0.25.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4df293303e8a52e1df7984ac1415e195f5fcbf51e4bb7bda54557861a3954a08"
"checksum base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2015e3793554aa5b6007e3a72959e84c1070039e74f13dde08fa64afe1ddd892"
"checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c"
"checksum bit-vec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5b97c2c8e8bbb4251754f559df8af22fb264853c7d009084a576cdf12565089d"
"checksum bitflags 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dead7461c1127cf637931a1e50934eb6eee8bff2f74433ac7909e9afcee04a3"
"checksum bitflags 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4f67931368edf3a9a51d29886d245f1c3db2f1ef0dcc9e35ff70341b78c10d23"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum blastfig 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "09640e0509d97d5cdff03a9f5daf087a8e04c735c3b113a75139634a19cfc7b2"
"checksum bloomchain 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3f421095d2a76fc24cd3fb3f912b90df06be7689912b1bdb423caefae59c258d"
"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
@ -2183,7 +2365,10 @@ dependencies = [
"checksum clippy_lints 0.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "ce96ec05bfe018a0d5d43da115e54850ea2217981ff0f2e462780ab9d594651a"
"checksum cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90266f45846f14a1e986c77d1e9c2626b8c342ed806fe60241ec38cc8697b245"
"checksum cookie 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d53b80dde876f47f03cda35303e368a79b91c70b0d65ecba5fd5280944a08591"
"checksum crossbeam 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "fb974f835e90390c5f9dfac00f05b06dc117299f5ea4e85fbc7bb443af4911cc"
"checksum core-foundation 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "20a6d0448d3a99d977ae4a2aa5a98d886a923e863e81ad9ff814645b6feb3bbd"
"checksum core-foundation-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "05eed248dc504a5391c63794fe4fb64f46f071280afaa1b73308f3c0ce4574c5"
"checksum crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0c5ea215664ca264da8a9d9c3be80d2eaf30923c259d03e870388eb927508f97"
"checksum crypt32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e34988f7e069e0b2f3bfc064295161e489b2d4e04a2e4248fb94360cdf00b4ec"
"checksum ctrlc 1.1.1 (git+https://github.com/ethcore/rust-ctrlc.git)" = "<none>"
"checksum daemonize 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "271ec51b7e0bee92f0d04601422c73eb76ececf197026711c97ad25038a010cf"
"checksum deque 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1614659040e711785ed8ea24219140654da1729f3ec8a47a9719d041112fe7bf"
@ -2196,14 +2381,16 @@ dependencies = [
"checksum fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b1ee15a7050e5580b3712877157068ea713b245b080ff302ae2ca973cfcd9baa"
"checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
"checksum futures 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bad0a2ac64b227fdc10c254051ae5af542cf19c9328704fd4092f7914196897"
"checksum futures-cpupool 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bb982bb25cd8fa5da6a8eb3a460354c984ff1113da82bcb4f0b0862b5795db82"
"checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
"checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518"
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
"checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1"
"checksum heapsize 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "abb306abb8d398e053cfb1b3e7b72c2f580be048b85745c52652954f8ad1439c"
"checksum hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2da7d3a34cf6406d9d700111b8eafafe9a251de41ae71d8052748259343b58"
"checksum httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46534074dbb80b070d60a5cb8ecadd8963a00a438ae1a95268850a7ef73b67ae"
"checksum hyper 0.10.0-a.0 (git+https://github.com/ethcore/hyper)" = "<none>"
"checksum hyper 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "eb27e8a3e8f17ac43ffa41bbda9cf5ad3f9f13ef66fa4873409d4902310275f7"
"checksum hyper 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bcb3fc65554155980167fb821d05c7c66177f92464976c0b676a19d9e03387a7"
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
"checksum igd 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c8c12b1795b8b168f577c45fa10379b3814dcb11b7ab702406001f0d63f40484"
"checksum isatty 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7408a548dc0e406b7912d9f84c261cc533c1866e047644a811c133c56041ac0c"
@ -2232,10 +2419,12 @@ dependencies = [
"checksum mio 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a637d1ca14eacae06296a008fa7ad955347e34efcb5891cfd8ba05491a37907e"
"checksum mio 0.6.0-dev (git+https://github.com/ethcore/mio?branch=timer-fix)" = "<none>"
"checksum mio 0.6.1 (git+https://github.com/ethcore/mio)" = "<none>"
"checksum mio 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "410a1a0ff76f5a226f1e4e3ff1756128e65cd30166e39c3892283e2ac09d5b67"
"checksum miow 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d5bfc6782530ac8ace97af10a540054a37126b63b0702ddaaa243b73b5745b9a"
"checksum msdos_time 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c04b68cc63a8480fb2550343695f7be72effdec953a9d4508161c3e69041c7d8"
"checksum nanomsg 0.5.1 (git+https://github.com/ethcore/nanomsg.rs.git)" = "<none>"
"checksum nanomsg-sys 0.5.0 (git+https://github.com/ethcore/nanomsg.rs.git)" = "<none>"
"checksum native-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aa4e52995154bb6f0b41e4379a279482c9387c1632e3798ba4e511ef8c54ee09"
"checksum net2 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "6a816012ca11cb47009693c1e0c6130e26d39e4d97ee2a13c50e868ec83e3204"
"checksum nix 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f05c2fc965fc1cd6b73fa57fa7b89f288178737f2f3ce9e63e4a6a141189000e"
"checksum nix 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a7bb1da2be7da3cbffda73fc681d509ffd9e665af478d2bee1907cee0bc64b2"
@ -2250,9 +2439,12 @@ dependencies = [
"checksum num-rational 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "48cdcc9ff4ae2a8296805ac15af88b3d88ce62128ded0cb74ffb63a587502a84"
"checksum num-traits 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "51eab148f171aefad295f8cece636fc488b9b392ef544da31ea4b8ef6b9e9c39"
"checksum num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "51fedae97a05f7353612fe017ab705a37e6db8f4d67c5c6fe739a9e70d6eed09"
"checksum num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55aabf4e2d6271a2e4e4c0f2ea1f5b07cc589cc1a9e9213013b54a76678ca4f3"
"checksum number_prefix 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "084d05f4bf60621a9ac9bde941a410df548f4de9545f06e5ee9d3aef4b97cd77"
"checksum odds 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "b28c06e81b0f789122d415d6394b5fe849bde8067469f4c2980d3cdc10c78ec1"
"checksum ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c"
"checksum openssl 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "12be61c7eaa23228316ff02c39807e4c1b1af84ba81420f19fd58dade304b25c"
"checksum openssl-sys 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d2845e841700e7b04282ceaa115407ea84e0db918ae689ad9ceb6f06fa6046bd"
"checksum owning_ref 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8d91377085359426407a287ab16884a0111ba473aa6844ff01d4ec20ce3d75e7"
"checksum parity-dapps-glue 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "98378dec0a185da2b7180308752f0bad73aaa949c3e0a3b0528d0e067945f7ab"
"checksum parity-ui-precompiled 1.4.0 (git+https://github.com/ethcore/js-precompiled.git)" = "<none>"
@ -2262,6 +2454,7 @@ dependencies = [
"checksum phf_codegen 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8af7ae7c3f75a502292b491e5cc0a1f69e3407744abe6e57e2a3b712bb82f01d"
"checksum phf_generator 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "db005608fd99800c8c74106a7c894cf582055b689aa14a79462cefdcb7dc1cc3"
"checksum phf_shared 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "fee4d039930e4f45123c9b15976cf93a499847b6483dc09c42ea0ec4940f2aa6"
"checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa"
"checksum podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e5422a1ee1bc57cc47ae717b0137314258138f38fd5f3cea083f43a9725383a0"
"checksum primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0e31b86efadeaeb1235452171a66689682783149a6249ff334a2c5d8218d00a4"
"checksum primal-bit 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "464a91febc06166783d4f5ba3577b5ed8dda8e421012df80bfe48a971ed7be8f"
@ -2278,7 +2471,7 @@ dependencies = [
"checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea"
"checksum regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)" = "b4329b8928a284580a1c63ec9d846b12f6d3472317243ff7077aff11f23f2b29"
"checksum regex-syntax 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "841591b1e05609a643e3b4d0045fce04f701daba7151ddcd3ad47b080693d5a9"
"checksum ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0d2f6547bf9640f1d3cc4e771f82374ec8fd237c17eeb3ff5cd5ccbe22377a09"
"checksum reqwest 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83186fee0d4dbeb95e610b77b05b05cf5b31703dd375222acb74c3dff4be957c"
"checksum rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
"checksum rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)" = "<none>"
"checksum rotor 0.6.3 (git+https://github.com/ethcore/rotor)" = "<none>"
@ -2287,15 +2480,20 @@ dependencies = [
"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a"
"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
"checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
"checksum rustls 0.1.2 (git+https://github.com/ctz/rustls)" = "<none>"
"checksum schannel 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "167852e03fcd0029c3ddebb5afb0715b2996f6e262b2c2aceaa7cd84edd4b158"
"checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d"
"checksum secur32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3f412dfa83308d893101dd59c10d6fda8283465976c28c287c5c855bf8d216bc"
"checksum security-framework 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "52186fcf3b391c9f0ccdce9a2ac708f7cc81b3f89e149b34bd9279fb1b23f9fa"
"checksum security-framework-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c84067e6297c1f09514a8666d8bbc1268817ec4a6c0f30f12f6201e1f34bd2a1"
"checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
"checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f"
"checksum semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae2ff60ecdb19c255841c066cbfa5f8c2a4ada1eb3ae47c77ab6667128da71f5"
"checksum semver-parser 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e88e43a5a74dd2a11707f9c21dfd4a423c66bd871df813227bb0a3e78f3a1ae9"
"checksum serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b1dfda9ebb31d29fa8b94d7eb3031a86a8dcec065f0fe268a30f98867bf45775"
"checksum serde 0.8.19 (registry+https://github.com/rust-lang/crates.io-index)" = "58a19c0871c298847e6b68318484685cd51fa5478c0c905095647540031356e5"
"checksum serde_codegen 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e422ae53d7933f59c6ff57e7b5870b5c9094b1f473f78ec33d89f8a692c3ec02"
"checksum serde_codegen_internals 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f877e2781ed0a323295d1c9f0e26556117b5a11489fc47b1848dfb98b3173d21"
"checksum serde_json 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0e10f8a9d94b06cf5d3bef66475f04c8ff90950f1be7004c357ff9472ccbaebc"
"checksum serde_urlencoded 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "53d4ebaa8d1d4f90d1b63dfca81ccd98ac20e1e479dbae393cbaf60f6fecd8d8"
"checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c"
"checksum shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72f20b8f3c060374edb8046591ba28f62448c369ccbdc7b02075103fb3a9e38d"
"checksum siphasher 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c44e42fa187b5a8782489cf7740cc27c3125806be2bf33563cf5e02e9533fcd"
@ -2323,6 +2521,7 @@ dependencies = [
"checksum thread_local 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0694f51610ef7cfac7a1b81de7f1602ee5356e76541bcd62c40e71933338cab1"
"checksum time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "3c7ec6d62a20df54e07ab3b78b9a3932972f4b7981de295563686849eb3989af"
"checksum tiny-keccak 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f7aef43048292ca0bae4ab32180e85f6202cf2816c2a210c396a84b99dab9270"
"checksum tokio-core 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "52416b3e937abac22a543a7f1c66bd37feb60137ff1ab42390fa02df85347e58"
"checksum toml 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "fcd27a04ca509aff336ba5eb2abc58d456f52c4ff64d9724d88acb85ead560b6"
"checksum toml 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a442dfc13508e603c3f763274361db7f79d7469a0e95c411cde53662ab30fc72"
"checksum traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "07eaeb7689bb7fca7ce15628319635758eda769fed481ecfe6686ddef2600616"
@ -2332,13 +2531,12 @@ dependencies = [
"checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
"checksum unicode-xid 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "36dff09cafb4ec7c8cf0023eb0b686cb6ce65499116a12201c9e11840ca01beb"
"checksum untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5d9bc0e6e73a10975d1fbff8ac3541e221181b0d8998351600fb5523de634c0d"
"checksum url 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afe9ec54bc4db14bc8744b7fed060d785ac756791450959b2248443319d5b119"
"checksum user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ef4711d107b21b410a3a974b1204d9accc8b10dad75d8324b5d755de1617d47"
"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
"checksum vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0795a11576d29ae80525a3fda315bf7b534f8feb9d34101e5fe63fb95bb2fd24"
"checksum vergen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "56b639f935488eb40f06d17c3e3bcc3054f6f75d264e187b1107c8d1cba8d31c"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "813503a5985585e0812d430cd1328ee322f47f66629c8ed4ecab939cf9e92f91"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum ws 0.5.3 (git+https://github.com/ethcore/ws-rs.git?branch=mio-upstream-stable)" = "<none>"

View File

@ -52,6 +52,7 @@ parity-rpc-client = { path = "rpc_client" }
ethcore-light = { path = "ethcore/light" }
parity-hash-fetch = { path = "hash-fetch" }
parity-updater = { path = "updater" }
parity-reactor = { path = "util/reactor" }
[target.'cfg(windows)'.dependencies]
winapi = "0.2"

View File

@ -12,6 +12,7 @@ build = "build.rs"
rand = "0.3.14"
log = "0.3"
env_logger = "0.3"
futures = "0.1"
jsonrpc-core = { git = "https://github.com/ethcore/jsonrpc.git" }
jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc.git" }
hyper = { default-features = false, git = "https://github.com/ethcore/hyper" }
@ -33,6 +34,7 @@ ethcore-util = { path = "../util" }
fetch = { path = "../util/fetch" }
parity-ui = { path = "./ui" }
parity-hash-fetch = { path = "../hash-fetch" }
parity-reactor = { path = "../util/reactor" }
clippy = { version = "0.0.103", optional = true}

View File

@ -23,7 +23,7 @@ use hyper::header::AccessControlAllowOrigin;
use api::types::{App, ApiError};
use api::response;
use apps::fetcher::ContentFetcher;
use apps::fetcher::Fetcher;
use handlers::extract_url;
use endpoint::{Endpoint, Endpoints, Handler, EndpointPath};
@ -33,11 +33,11 @@ use jsonrpc_http_server::cors;
pub struct RestApi {
cors_domains: Option<Vec<AccessControlAllowOrigin>>,
endpoints: Arc<Endpoints>,
fetcher: Arc<ContentFetcher>,
fetcher: Arc<Fetcher>,
}
impl RestApi {
pub fn new(cors_domains: Vec<String>, endpoints: Arc<Endpoints>, fetcher: Arc<ContentFetcher>) -> Box<Endpoint> {
pub fn new(cors_domains: Vec<String>, endpoints: Arc<Endpoints>, fetcher: Arc<Fetcher>) -> Box<Endpoint> {
Box::new(RestApi {
cors_domains: Some(cors_domains.into_iter().map(AccessControlAllowOrigin::Value).collect()),
endpoints: endpoints,

View File

@ -1,445 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Fetchable Dapps support.
//! Manages downloaded (cached) Dapps and downloads them when necessary.
//! Uses `URLHint` to resolve addresses into Dapps bundle file location.
use zip;
use std::{fs, env, fmt};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use rustc_serialize::hex::FromHex;
use hash_fetch::urlhint::{URLHintContract, URLHint, URLHintResult};
use hyper;
use hyper::status::StatusCode;
use random_filename;
use SyncStatus;
use util::{Mutex, H256};
use util::sha3::sha3;
use page::{LocalPageEndpoint, PageCache};
use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
use endpoint::{Endpoint, EndpointPath, Handler};
use apps::cache::{ContentCache, ContentStatus};
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
/// Limit of cached dapps/content
const MAX_CACHED_DAPPS: usize = 20;
pub struct ContentFetcher<R: URLHint = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
cache: Arc<Mutex<ContentCache>>,
sync: Arc<SyncStatus>,
embeddable_on: Option<(String, u16)>,
}
impl<R: URLHint> Drop for ContentFetcher<R> {
fn drop(&mut self) {
// Clear cache path
let _ = fs::remove_dir_all(&self.dapps_path);
}
}
impl<R: URLHint> ContentFetcher<R> {
pub fn new(resolver: R, sync_status: Arc<SyncStatus>, embeddable_on: Option<(String, u16)>) -> Self {
let mut dapps_path = env::temp_dir();
dapps_path.push(random_filename());
ContentFetcher {
dapps_path: dapps_path,
resolver: resolver,
sync: sync_status,
cache: Arc::new(Mutex::new(ContentCache::default())),
embeddable_on: embeddable_on,
}
}
fn still_syncing(address: Option<(String, u16)>) -> Box<Handler> {
Box::new(ContentHandler::error(
StatusCode::ServiceUnavailable,
"Sync In Progress",
"Your node is still syncing. We cannot resolve any content before it's fully synced.",
Some("<a href=\"javascript:window.location.reload()\">Refresh</a>"),
address,
))
}
#[cfg(test)]
fn set_status(&self, content_id: &str, status: ContentStatus) {
self.cache.lock().insert(content_id.to_owned(), status);
}
pub fn contains(&self, content_id: &str) -> bool {
{
let mut cache = self.cache.lock();
// Check if we already have the app
if cache.get(content_id).is_some() {
return true;
}
}
// fallback to resolver
if let Ok(content_id) = content_id.from_hex() {
// else try to resolve the app_id
let has_content = self.resolver.resolve(content_id).is_some();
// if there is content or we are syncing return true
has_content || self.sync.is_major_importing()
} else {
false
}
}
pub fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
let mut cache = self.cache.lock();
let content_id = path.app_id.clone();
let (new_status, handler) = {
let status = cache.get(&content_id);
match status {
// Just serve the content
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_async_handler(path, control))
},
// Content is already being fetched
Some(&mut ContentStatus::Fetching(ref fetch_control)) => {
trace!(target: "dapps", "Content fetching in progress. Waiting...");
(None, fetch_control.to_async_handler(path, control))
},
// We need to start fetching the content
None => {
trace!(target: "dapps", "Content unavailable. Fetching... {:?}", content_id);
let content_hex = content_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let content = self.resolver.resolve(content_hex);
let cache = self.cache.clone();
let id = content_id.clone();
let on_done = move |result: Option<LocalPageEndpoint>| {
let mut cache = cache.lock();
match result {
Some(endpoint) => {
cache.insert(id.clone(), ContentStatus::Ready(endpoint));
},
// In case of error
None => {
cache.remove(&id);
},
}
};
match content {
// Don't serve dapps if we are still syncing (but serve content)
Some(URLHintResult::Dapp(_)) if self.sync.is_major_importing() => {
(None, Self::still_syncing(self.embeddable_on.clone()))
},
Some(URLHintResult::Dapp(dapp)) => {
let (handler, fetch_control) = ContentFetcherHandler::new(
dapp.url(),
path,
control,
DappInstaller {
id: content_id.clone(),
dapps_path: self.dapps_path.clone(),
on_done: Box::new(on_done),
embeddable_on: self.embeddable_on.clone(),
},
self.embeddable_on.clone(),
);
(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
},
Some(URLHintResult::Content(content)) => {
let (handler, fetch_control) = ContentFetcherHandler::new(
content.url,
path,
control,
ContentInstaller {
id: content_id.clone(),
mime: content.mime,
content_path: self.dapps_path.clone(),
on_done: Box::new(on_done),
},
self.embeddable_on.clone(),
);
(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
},
None if self.sync.is_major_importing() => {
(None, Self::still_syncing(self.embeddable_on.clone()))
},
None => {
// This may happen when sync status changes in between
// `contains` and `to_handler`
(None, Box::new(ContentHandler::error(
StatusCode::NotFound,
"Resource Not Found",
"Requested resource was not found.",
None,
self.embeddable_on.clone(),
)) as Box<Handler>)
},
}
},
}
};
if let Some(status) = new_status {
cache.clear_garbage(MAX_CACHED_DAPPS);
cache.insert(content_id, status);
}
handler
}
}
#[derive(Debug)]
pub enum ValidationError {
Io(io::Error),
Zip(zip::result::ZipError),
InvalidContentId,
ManifestNotFound,
ManifestSerialization(String),
HashMismatch { expected: H256, got: H256, },
}
impl fmt::Display for ValidationError {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match *self {
ValidationError::Io(ref io) => write!(f, "Unexpected IO error occured: {:?}", io),
ValidationError::Zip(ref zip) => write!(f, "Unable to read ZIP archive: {:?}", zip),
ValidationError::InvalidContentId => write!(f, "ID is invalid. It should be 256 bits keccak hash of content."),
ValidationError::ManifestNotFound => write!(f, "Downloaded Dapp bundle did not contain valid manifest.json file."),
ValidationError::ManifestSerialization(ref err) => {
write!(f, "There was an error during Dapp Manifest serialization: {:?}", err)
},
ValidationError::HashMismatch { ref expected, ref got } => {
write!(f, "Hash of downloaded content did not match. Expected:{:?}, Got:{:?}.", expected, got)
},
}
}
}
impl From<io::Error> for ValidationError {
fn from(err: io::Error) -> Self {
ValidationError::Io(err)
}
}
impl From<zip::result::ZipError> for ValidationError {
fn from(err: zip::result::ZipError) -> Self {
ValidationError::Zip(err)
}
}
struct ContentInstaller {
id: String,
mime: String,
content_path: PathBuf,
on_done: Box<Fn(Option<LocalPageEndpoint>) + Send>,
}
impl ContentValidator for ContentInstaller {
type Error = ValidationError;
fn validate_and_install(&self, path: PathBuf) -> Result<LocalPageEndpoint, ValidationError> {
let validate = || {
// Create dir
try!(fs::create_dir_all(&self.content_path));
// Validate hash
let mut file_reader = io::BufReader::new(try!(fs::File::open(&path)));
let hash = try!(sha3(&mut file_reader));
let id = try!(self.id.as_str().parse().map_err(|_| ValidationError::InvalidContentId));
if id != hash {
return Err(ValidationError::HashMismatch {
expected: id,
got: hash,
});
}
// And prepare path for a file
let filename = path.file_name().expect("We always fetch a file.");
let mut content_path = self.content_path.clone();
content_path.push(&filename);
if content_path.exists() {
try!(fs::remove_dir_all(&content_path))
}
try!(fs::copy(&path, &content_path));
Ok(LocalPageEndpoint::single_file(content_path, self.mime.clone(), PageCache::Enabled))
};
// Make sure to always call on_done (even in case of errors)!
let result = validate();
(self.on_done)(result.as_ref().ok().cloned());
result
}
}
struct DappInstaller {
id: String,
dapps_path: PathBuf,
on_done: Box<Fn(Option<LocalPageEndpoint>) + Send>,
embeddable_on: Option<(String, u16)>,
}
impl DappInstaller {
fn find_manifest(zip: &mut zip::ZipArchive<fs::File>) -> Result<(Manifest, PathBuf), ValidationError> {
for i in 0..zip.len() {
let mut file = try!(zip.by_index(i));
if !file.name().ends_with(MANIFEST_FILENAME) {
continue;
}
// try to read manifest
let mut manifest = String::new();
let manifest = file
.read_to_string(&mut manifest).ok()
.and_then(|_| deserialize_manifest(manifest).ok());
if let Some(manifest) = manifest {
let mut manifest_location = PathBuf::from(file.name());
manifest_location.pop(); // get rid of filename
return Ok((manifest, manifest_location));
}
}
Err(ValidationError::ManifestNotFound)
}
fn dapp_target_path(&self, manifest: &Manifest) -> PathBuf {
let mut target = self.dapps_path.clone();
target.push(&manifest.id);
target
}
}
impl ContentValidator for DappInstaller {
type Error = ValidationError;
fn validate_and_install(&self, path: PathBuf) -> Result<LocalPageEndpoint, ValidationError> {
trace!(target: "dapps", "Opening dapp bundle at {:?}", path);
let validate = || {
let mut file_reader = io::BufReader::new(try!(fs::File::open(path)));
let hash = try!(sha3(&mut file_reader));
let id = try!(self.id.as_str().parse().map_err(|_| ValidationError::InvalidContentId));
if id != hash {
return Err(ValidationError::HashMismatch {
expected: id,
got: hash,
});
}
let file = file_reader.into_inner();
// Unpack archive
let mut zip = try!(zip::ZipArchive::new(file));
// First find manifest file
let (mut manifest, manifest_dir) = try!(Self::find_manifest(&mut zip));
// Overwrite id to match hash
manifest.id = self.id.clone();
let target = self.dapp_target_path(&manifest);
// Remove old directory
if target.exists() {
warn!(target: "dapps", "Overwriting existing dapp: {}", manifest.id);
try!(fs::remove_dir_all(target.clone()));
}
// Unpack zip
for i in 0..zip.len() {
let mut file = try!(zip.by_index(i));
// TODO [todr] Check if it's consistent on windows.
let is_dir = file.name().chars().rev().next() == Some('/');
let file_path = PathBuf::from(file.name());
let location_in_manifest_base = file_path.strip_prefix(&manifest_dir);
// Create files that are inside manifest directory
if let Ok(location_in_manifest_base) = location_in_manifest_base {
let p = target.join(location_in_manifest_base);
// Check if it's a directory
if is_dir {
try!(fs::create_dir_all(p));
} else {
let mut target = try!(fs::File::create(p));
try!(io::copy(&mut file, &mut target));
}
}
}
// Write manifest
let manifest_str = try!(serialize_manifest(&manifest).map_err(ValidationError::ManifestSerialization));
let manifest_path = target.join(MANIFEST_FILENAME);
let mut manifest_file = try!(fs::File::create(manifest_path));
try!(manifest_file.write_all(manifest_str.as_bytes()));
// Create endpoint
let endpoint = LocalPageEndpoint::new(target, manifest.clone().into(), PageCache::Enabled, self.embeddable_on.clone());
Ok(endpoint)
};
let result = validate();
(self.on_done)(result.as_ref().ok().cloned());
result
}
}
#[cfg(test)]
mod tests {
use std::env;
use std::sync::Arc;
use util::Bytes;
use hash_fetch::urlhint::{URLHint, URLHintResult};
use apps::cache::ContentStatus;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use super::ContentFetcher;
struct FakeResolver;
impl URLHint for FakeResolver {
fn resolve(&self, _id: Bytes) -> Option<URLHintResult> {
None
}
}
#[test]
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = ContentFetcher::new(FakeResolver, Arc::new(|| false), None);
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),
author: "".into(),
icon_url: "".into(),
}, Default::default(), None);
// when
fetcher.set_status("test", ContentStatus::Ready(handler));
fetcher.set_status("test2", ContentStatus::Fetching(Default::default()));
// then
assert_eq!(fetcher.contains("test"), true);
assert_eq!(fetcher.contains("test2"), true);
assert_eq!(fetcher.contains("test3"), false);
}
}

View File

@ -0,0 +1,255 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use zip;
use std::{fs, fmt};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use fetch::{self, Mime};
use util::H256;
use util::sha3::sha3;
use page::{LocalPageEndpoint, PageCache};
use handlers::ContentValidator;
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
type OnDone = Box<Fn(Option<LocalPageEndpoint>) + Send>;
fn write_response_and_check_hash(
id: &str,
mut content_path: PathBuf,
filename: &str,
response: fetch::Response
) -> Result<(fs::File, PathBuf), ValidationError> {
// try to parse id
let id = try!(id.parse().map_err(|_| ValidationError::InvalidContentId));
// check if content exists
if content_path.exists() {
warn!(target: "dapps", "Overwriting existing content at 0x{:?}", id);
try!(fs::remove_dir_all(&content_path))
}
// create directory
try!(fs::create_dir_all(&content_path));
// append filename
content_path.push(filename);
// Now write the response
let mut file = io::BufWriter::new(try!(fs::File::create(&content_path)));
let mut reader = io::BufReader::new(response);
try!(io::copy(&mut reader, &mut file));
try!(file.flush());
// Validate hash
// TODO [ToDr] calculate sha3 in-flight while reading the response
let mut file = io::BufReader::new(try!(fs::File::open(&content_path)));
let hash = try!(sha3(&mut file));
if id == hash {
Ok((file.into_inner(), content_path))
} else {
Err(ValidationError::HashMismatch {
expected: id,
got: hash,
})
}
}
pub struct Content {
id: String,
mime: Mime,
content_path: PathBuf,
on_done: OnDone,
}
impl Content {
pub fn new(id: String, mime: Mime, content_path: PathBuf, on_done: OnDone) -> Self {
Content {
id: id,
mime: mime,
content_path: content_path,
on_done: on_done,
}
}
}
impl ContentValidator for Content {
type Error = ValidationError;
fn validate_and_install(&self, response: fetch::Response) -> Result<LocalPageEndpoint, ValidationError> {
let validate = |content_path: PathBuf| {
// Create dir
let (_, content_path) = try!(write_response_and_check_hash(self.id.as_str(), content_path.clone(), self.id.as_str(), response));
Ok(LocalPageEndpoint::single_file(content_path, self.mime.clone(), PageCache::Enabled))
};
// Prepare path for a file
let content_path = self.content_path.join(&self.id);
// Make sure to always call on_done (even in case of errors)!
let result = validate(content_path.clone());
// remove the file if there was an error
if result.is_err() {
// Ignore errors since the file might not exist
let _ = fs::remove_dir_all(&content_path);
}
(self.on_done)(result.as_ref().ok().cloned());
result
}
}
pub struct Dapp {
id: String,
dapps_path: PathBuf,
on_done: OnDone,
embeddable_on: Option<(String, u16)>,
}
impl Dapp {
pub fn new(id: String, dapps_path: PathBuf, on_done: OnDone, embeddable_on: Option<(String, u16)>) -> Self {
Dapp {
id: id,
dapps_path: dapps_path,
on_done: on_done,
embeddable_on: embeddable_on,
}
}
fn find_manifest(zip: &mut zip::ZipArchive<fs::File>) -> Result<(Manifest, PathBuf), ValidationError> {
for i in 0..zip.len() {
let mut file = try!(zip.by_index(i));
if !file.name().ends_with(MANIFEST_FILENAME) {
continue;
}
// try to read manifest
let mut manifest = String::new();
let manifest = file
.read_to_string(&mut manifest).ok()
.and_then(|_| deserialize_manifest(manifest).ok());
if let Some(manifest) = manifest {
let mut manifest_location = PathBuf::from(file.name());
manifest_location.pop(); // get rid of filename
return Ok((manifest, manifest_location));
}
}
Err(ValidationError::ManifestNotFound)
}
}
impl ContentValidator for Dapp {
type Error = ValidationError;
fn validate_and_install(&self, response: fetch::Response) -> Result<LocalPageEndpoint, ValidationError> {
let validate = |dapp_path: PathBuf| {
let (file, zip_path) = try!(write_response_and_check_hash(self.id.as_str(), dapp_path.clone(), &format!("{}.zip", self.id), response));
trace!(target: "dapps", "Opening dapp bundle at {:?}", zip_path);
// Unpack archive
let mut zip = try!(zip::ZipArchive::new(file));
// First find manifest file
let (mut manifest, manifest_dir) = try!(Self::find_manifest(&mut zip));
// Overwrite id to match hash
manifest.id = self.id.clone();
// Unpack zip
for i in 0..zip.len() {
let mut file = try!(zip.by_index(i));
let is_dir = file.name().chars().rev().next() == Some('/');
let file_path = PathBuf::from(file.name());
let location_in_manifest_base = file_path.strip_prefix(&manifest_dir);
// Create files that are inside manifest directory
if let Ok(location_in_manifest_base) = location_in_manifest_base {
let p = dapp_path.join(location_in_manifest_base);
// Check if it's a directory
if is_dir {
try!(fs::create_dir_all(p));
} else {
let mut target = try!(fs::File::create(p));
try!(io::copy(&mut file, &mut target));
}
}
}
// Remove zip
try!(fs::remove_file(&zip_path));
// Write manifest
let manifest_str = try!(serialize_manifest(&manifest).map_err(ValidationError::ManifestSerialization));
let manifest_path = dapp_path.join(MANIFEST_FILENAME);
let mut manifest_file = try!(fs::File::create(manifest_path));
try!(manifest_file.write_all(manifest_str.as_bytes()));
// Create endpoint
let endpoint = LocalPageEndpoint::new(dapp_path, manifest.clone().into(), PageCache::Enabled, self.embeddable_on.clone());
Ok(endpoint)
};
// Prepare directory for dapp
let target = self.dapps_path.join(&self.id);
// Validate the dapp
let result = validate(target.clone());
// remove the file if there was an error
if result.is_err() {
// Ignore errors since the file might not exist
let _ = fs::remove_dir_all(&target);
}
(self.on_done)(result.as_ref().ok().cloned());
result
}
}
#[derive(Debug)]
pub enum ValidationError {
Io(io::Error),
Zip(zip::result::ZipError),
InvalidContentId,
ManifestNotFound,
ManifestSerialization(String),
HashMismatch { expected: H256, got: H256, },
}
impl fmt::Display for ValidationError {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match *self {
ValidationError::Io(ref io) => write!(f, "Unexpected IO error occured: {:?}", io),
ValidationError::Zip(ref zip) => write!(f, "Unable to read ZIP archive: {:?}", zip),
ValidationError::InvalidContentId => write!(f, "ID is invalid. It should be 256 bits keccak hash of content."),
ValidationError::ManifestNotFound => write!(f, "Downloaded Dapp bundle did not contain valid manifest.json file."),
ValidationError::ManifestSerialization(ref err) => {
write!(f, "There was an error during Dapp Manifest serialization: {:?}", err)
},
ValidationError::HashMismatch { ref expected, ref got } => {
write!(f, "Hash of downloaded content did not match. Expected:{:?}, Got:{:?}.", expected, got)
},
}
}
}
impl From<io::Error> for ValidationError {
fn from(err: io::Error) -> Self {
ValidationError::Io(err)
}
}
impl From<zip::result::ZipError> for ValidationError {
fn from(err: zip::result::ZipError) -> Self {
ValidationError::Zip(err)
}
}

View File

@ -0,0 +1,265 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Fetchable Dapps support.
//! Manages downloaded (cached) Dapps and downloads them when necessary.
//! Uses `URLHint` to resolve addresses into Dapps bundle file location.
mod installers;
use std::{fs, env};
use std::path::PathBuf;
use std::sync::Arc;
use rustc_serialize::hex::FromHex;
use fetch::{Client as FetchClient, Fetch};
use hash_fetch::urlhint::{URLHintContract, URLHint, URLHintResult};
use parity_reactor::Remote;
use hyper;
use hyper::status::StatusCode;
use {SyncStatus, random_filename};
use util::Mutex;
use page::LocalPageEndpoint;
use handlers::{ContentHandler, ContentFetcherHandler};
use endpoint::{Endpoint, EndpointPath, Handler};
use apps::cache::{ContentCache, ContentStatus};
/// Limit of cached dapps/content
const MAX_CACHED_DAPPS: usize = 20;
pub trait Fetcher: Send + Sync + 'static {
fn contains(&self, content_id: &str) -> bool;
fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler>;
}
pub struct ContentFetcher<F: Fetch = FetchClient, R: URLHint + Send + Sync + 'static = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
cache: Arc<Mutex<ContentCache>>,
sync: Arc<SyncStatus>,
embeddable_on: Option<(String, u16)>,
remote: Remote,
fetch: F,
}
impl<R: URLHint + Send + Sync + 'static, F: Fetch> Drop for ContentFetcher<F, R> {
fn drop(&mut self) {
// Clear cache path
let _ = fs::remove_dir_all(&self.dapps_path);
}
}
impl<R: URLHint + Send + Sync + 'static, F: Fetch> ContentFetcher<F, R> {
pub fn new(resolver: R, sync_status: Arc<SyncStatus>, embeddable_on: Option<(String, u16)>, remote: Remote, fetch: F) -> Self {
let mut dapps_path = env::temp_dir();
dapps_path.push(random_filename());
ContentFetcher {
dapps_path: dapps_path,
resolver: resolver,
sync: sync_status,
cache: Arc::new(Mutex::new(ContentCache::default())),
embeddable_on: embeddable_on,
remote: remote,
fetch: fetch,
}
}
fn still_syncing(address: Option<(String, u16)>) -> Box<Handler> {
Box::new(ContentHandler::error(
StatusCode::ServiceUnavailable,
"Sync In Progress",
"Your node is still syncing. We cannot resolve any content before it's fully synced.",
Some("<a href=\"javascript:window.location.reload()\">Refresh</a>"),
address,
))
}
#[cfg(test)]
fn set_status(&self, content_id: &str, status: ContentStatus) {
self.cache.lock().insert(content_id.to_owned(), status);
}
}
impl<R: URLHint + Send + Sync + 'static, F: Fetch> Fetcher for ContentFetcher<F, R> {
fn contains(&self, content_id: &str) -> bool {
{
let mut cache = self.cache.lock();
// Check if we already have the app
if cache.get(content_id).is_some() {
return true;
}
}
// fallback to resolver
if let Ok(content_id) = content_id.from_hex() {
// else try to resolve the app_id
let has_content = self.resolver.resolve(content_id).is_some();
// if there is content or we are syncing return true
has_content || self.sync.is_major_importing()
} else {
false
}
}
fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
let mut cache = self.cache.lock();
let content_id = path.app_id.clone();
let (new_status, handler) = {
let status = cache.get(&content_id);
match status {
// Just serve the content
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_async_handler(path, control))
},
// Content is already being fetched
Some(&mut ContentStatus::Fetching(ref fetch_control)) => {
trace!(target: "dapps", "Content fetching in progress. Waiting...");
(None, fetch_control.to_async_handler(path, control))
},
// We need to start fetching the content
None => {
trace!(target: "dapps", "Content unavailable. Fetching... {:?}", content_id);
let content_hex = content_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let content = self.resolver.resolve(content_hex);
let cache = self.cache.clone();
let id = content_id.clone();
let on_done = move |result: Option<LocalPageEndpoint>| {
let mut cache = cache.lock();
match result {
Some(endpoint) => cache.insert(id.clone(), ContentStatus::Ready(endpoint)),
// In case of error
None => cache.remove(&id),
};
};
match content {
// Don't serve dapps if we are still syncing (but serve content)
Some(URLHintResult::Dapp(_)) if self.sync.is_major_importing() => {
(None, Self::still_syncing(self.embeddable_on.clone()))
},
Some(URLHintResult::Dapp(dapp)) => {
let (handler, fetch_control) = ContentFetcherHandler::new(
dapp.url(),
path,
control,
installers::Dapp::new(
content_id.clone(),
self.dapps_path.clone(),
Box::new(on_done),
self.embeddable_on.clone(),
),
self.embeddable_on.clone(),
self.remote.clone(),
self.fetch.clone(),
);
(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
},
Some(URLHintResult::Content(content)) => {
let (handler, fetch_control) = ContentFetcherHandler::new(
content.url,
path,
control,
installers::Content::new(
content_id.clone(),
content.mime,
self.dapps_path.clone(),
Box::new(on_done),
),
self.embeddable_on.clone(),
self.remote.clone(),
self.fetch.clone(),
);
(Some(ContentStatus::Fetching(fetch_control)), Box::new(handler) as Box<Handler>)
},
None if self.sync.is_major_importing() => {
(None, Self::still_syncing(self.embeddable_on.clone()))
},
None => {
// This may happen when sync status changes in between
// `contains` and `to_handler`
(None, Box::new(ContentHandler::error(
StatusCode::NotFound,
"Resource Not Found",
"Requested resource was not found.",
None,
self.embeddable_on.clone(),
)) as Box<Handler>)
},
}
},
}
};
if let Some(status) = new_status {
cache.clear_garbage(MAX_CACHED_DAPPS);
cache.insert(content_id, status);
}
handler
}
}
#[cfg(test)]
mod tests {
use std::env;
use std::sync::Arc;
use util::Bytes;
use fetch::Client;
use hash_fetch::urlhint::{URLHint, URLHintResult};
use parity_reactor::Remote;
use apps::cache::ContentStatus;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use super::{ContentFetcher, Fetcher};
struct FakeResolver;
impl URLHint for FakeResolver {
fn resolve(&self, _id: Bytes) -> Option<URLHintResult> {
None
}
}
#[test]
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = ContentFetcher::new(FakeResolver, Arc::new(|| false), None, Remote::new_sync(), Client::new().unwrap());
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),
author: "".into(),
icon_url: "".into(),
}, Default::default(), None);
// when
fetcher.set_status("test", ContentStatus::Ready(handler));
fetcher.set_status("test2", ContentStatus::Fetching(Default::default()));
// then
assert_eq!(fetcher.contains("test"), true);
assert_eq!(fetcher.contains("test2"), true);
assert_eq!(fetcher.contains("test3"), false);
}
}

View File

@ -17,7 +17,10 @@
use endpoint::{Endpoints, Endpoint};
use page::PageEndpoint;
use proxypac::ProxyPac;
use web::Web;
use fetch::Fetch;
use parity_dapps::WebApp;
use parity_reactor::Remote;
mod cache;
mod fs;
@ -27,22 +30,24 @@ pub mod manifest;
extern crate parity_ui;
pub const HOME_PAGE: &'static str = "home";
pub const DAPPS_DOMAIN : &'static str = ".parity";
pub const RPC_PATH : &'static str = "rpc";
pub const API_PATH : &'static str = "api";
pub const UTILS_PATH : &'static str = "parity-utils";
pub const DAPPS_DOMAIN: &'static str = ".parity";
pub const RPC_PATH: &'static str = "rpc";
pub const API_PATH: &'static str = "api";
pub const UTILS_PATH: &'static str = "parity-utils";
pub const WEB_PATH: &'static str = "web";
pub fn utils() -> Box<Endpoint> {
Box::new(PageEndpoint::with_prefix(parity_ui::App::default(), UTILS_PATH.to_owned()))
}
pub fn all_endpoints(dapps_path: String, signer_address: Option<(String, u16)>) -> Endpoints {
pub fn all_endpoints<F: Fetch>(dapps_path: String, signer_address: Option<(String, u16)>, remote: Remote, fetch: F) -> Endpoints {
// fetch fs dapps at first to avoid overwriting builtins
let mut pages = fs::local_endpoints(dapps_path, signer_address.clone());
// NOTE [ToDr] Dapps will be currently embeded on 8180
insert::<parity_ui::App>(&mut pages, "ui", Embeddable::Yes(signer_address.clone()));
pages.insert("proxy".into(), ProxyPac::boxed(signer_address));
pages.insert(WEB_PATH.into(), Web::boxed(remote, fetch));
pages
}

View File

@ -16,13 +16,14 @@
//! Hyper Server Handler that fetches a file during a request (proxy).
use std::{fs, fmt};
use std::path::PathBuf;
use std::fmt;
use std::sync::{mpsc, Arc};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Instant, Duration};
use fetch::{self, Fetch};
use futures::Future;
use parity_reactor::Remote;
use util::Mutex;
use fetch::{Client, Fetch, FetchResult};
use hyper::{server, Decoder, Encoder, Next, Method, Control};
use hyper::net::HttpStream;
@ -39,7 +40,7 @@ enum FetchState {
Waiting,
NotStarted(String),
Error(ContentHandler),
InProgress(mpsc::Receiver<FetchResult>),
InProgress(mpsc::Receiver<FetchState>),
Done(LocalPageEndpoint, Box<PageHandlerWaiting>),
}
@ -48,10 +49,10 @@ enum WaitResult {
Done(LocalPageEndpoint),
}
pub trait ContentValidator {
pub trait ContentValidator: Send + 'static {
type Error: fmt::Debug + fmt::Display;
fn validate_and_install(&self, path: PathBuf) -> Result<LocalPageEndpoint, Self::Error>;
fn validate_and_install(&self, fetch::Response) -> Result<LocalPageEndpoint, Self::Error>;
}
pub struct FetchControl {
@ -160,33 +161,36 @@ impl server::Handler<HttpStream> for WaitingHandler {
}
}
pub struct ContentFetcherHandler<H: ContentValidator> {
pub struct ContentFetcherHandler<H: ContentValidator, F: Fetch> {
fetch_control: Arc<FetchControl>,
control: Option<Control>,
control: Control,
remote: Remote,
status: FetchState,
client: Option<Client>,
installer: H,
fetch: F,
installer: Option<H>,
path: EndpointPath,
uri: RequestUri,
embeddable_on: Option<(String, u16)>,
}
impl<H: ContentValidator> ContentFetcherHandler<H> {
impl<H: ContentValidator, F: Fetch> ContentFetcherHandler<H, F> {
pub fn new(
url: String,
path: EndpointPath,
control: Control,
handler: H,
embeddable_on: Option<(String, u16)>,
remote: Remote,
fetch: F,
) -> (Self, Arc<FetchControl>) {
let fetch_control = Arc::new(FetchControl::default());
let client = Client::default();
let handler = ContentFetcherHandler {
fetch_control: fetch_control.clone(),
control: Some(control),
client: Some(client),
control: control,
remote: remote,
fetch: fetch,
status: FetchState::NotStarted(url),
installer: handler,
installer: Some(handler),
path: path,
uri: RequestUri::default(),
embeddable_on: embeddable_on,
@ -195,41 +199,74 @@ impl<H: ContentValidator> ContentFetcherHandler<H> {
(handler, fetch_control)
}
fn close_client(client: &mut Option<Client>) {
client.take()
.expect("After client is closed we are going into write, hence we can never close it again")
.close();
}
fn fetch_content(&self, url: &str, installer: H) -> mpsc::Receiver<FetchState> {
let (tx, rx) = mpsc::channel();
let abort = self.fetch_control.abort.clone();
fn fetch_content(client: &mut Client, url: &str, abort: Arc<AtomicBool>, control: Control) -> Result<mpsc::Receiver<FetchResult>, String> {
client.request(url, abort, Box::new(move || {
trace!(target: "dapps", "Fetching finished.");
let control = self.control.clone();
let embeddable_on = self.embeddable_on.clone();
let uri = self.uri.clone();
let path = self.path.clone();
let future = self.fetch.fetch_with_abort(url, abort.into()).then(move |result| {
trace!(target: "dapps", "Fetching content finished. Starting validation: {:?}", result);
let new_state = match result {
Ok(response) => match installer.validate_and_install(response) {
Ok(endpoint) => {
trace!(target: "dapps", "Validation OK. Returning response.");
let mut handler = endpoint.to_page_handler(path);
handler.set_uri(&uri);
FetchState::Done(endpoint, handler)
},
Err(e) => {
trace!(target: "dapps", "Error while validating content: {:?}", e);
FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
"Invalid Dapp",
"Downloaded bundle does not contain a valid content.",
Some(&format!("{:?}", e)),
embeddable_on,
))
},
},
Err(e) => {
warn!(target: "dapps", "Unable to fetch content: {:?}", e);
FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
"Download Error",
"There was an error when fetching the content.",
Some(&format!("{:?}", e)),
embeddable_on,
))
},
};
// Content may be resolved when the connection is already dropped.
let _ = tx.send(new_state);
// Ignoring control errors
let _ = control.ready(Next::read());
})).map_err(|e| format!("{:?}", e))
Ok(()) as Result<(), ()>
});
// make sure to run within fetch thread pool.
let future = self.fetch.process(future);
// spawn to event loop
self.remote.spawn(future);
rx
}
}
impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<H> {
impl<H: ContentValidator, F: Fetch> server::Handler<HttpStream> for ContentFetcherHandler<H, F> {
fn on_request(&mut self, request: server::Request<HttpStream>) -> Next {
self.uri = request.uri().clone();
let installer = self.installer.take().expect("Installer always set initialy; installer used only in on_request; on_request invoked only once; qed");
let status = if let FetchState::NotStarted(ref url) = self.status {
Some(match *request.method() {
// Start fetching content
Method::Get => {
trace!(target: "dapps", "Fetching content from: {:?}", url);
let control = self.control.take().expect("on_request is called only once, thus control is always Some");
let client = self.client.as_mut().expect("on_request is called before client is closed.");
let fetch = Self::fetch_content(client, url, self.fetch_control.abort.clone(), control);
match fetch {
Ok(receiver) => FetchState::InProgress(receiver),
Err(e) => FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
"Unable To Start Content Download",
"Could not initialize download of the content. It might be a problem with the remote server.",
Some(&format!("{}", e)),
self.embeddable_on.clone(),
)),
}
let receiver = self.fetch_content(url, installer);
FetchState::InProgress(receiver)
},
// or return error
_ => FetchState::Error(ContentHandler::error(
@ -246,7 +283,6 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
self.fetch_control.set_status(&status);
self.status = status;
}
self.uri = request.uri().clone();
Next::read()
}
@ -263,50 +299,14 @@ impl<H: ContentValidator> server::Handler<HttpStream> for ContentFetcherHandler<
None,
self.embeddable_on.clone(),
);
Self::close_client(&mut self.client);
(Some(FetchState::Error(timeout)), Next::write())
},
FetchState::InProgress(ref receiver) => {
// Check if there is an answer
let rec = receiver.try_recv();
match rec {
// Unpack and validate
Ok(Ok(path)) => {
trace!(target: "dapps", "Fetching content finished. Starting validation ({:?})", path);
Self::close_client(&mut self.client);
// Unpack and verify
let state = match self.installer.validate_and_install(path.clone()) {
Err(e) => {
trace!(target: "dapps", "Error while validating content: {:?}", e);
FetchState::Error(ContentHandler::error(
StatusCode::BadGateway,
"Invalid Dapp",
"Downloaded bundle does not contain a valid content.",
Some(&format!("{:?}", e)),
self.embeddable_on.clone(),
))
},
Ok(endpoint) => {
let mut handler = endpoint.to_page_handler(self.path.clone());
handler.set_uri(&self.uri);
FetchState::Done(endpoint, handler)
},
};
// Remove temporary zip file
let _ = fs::remove_file(path);
(Some(state), Next::write())
},
Ok(Err(e)) => {
warn!(target: "dapps", "Unable to fetch content: {:?}", e);
let error = ContentHandler::error(
StatusCode::BadGateway,
"Download Error",
"There was an error when fetching the content.",
Some(&format!("{:?}", e)),
self.embeddable_on.clone(),
);
(Some(FetchState::Error(error)), Next::write())
},
// just return the new state
Ok(state) => (Some(state), Next::write()),
// wait some more
_ => (None, Next::wait())
}

View File

@ -61,6 +61,9 @@ extern crate parity_hash_fetch as hash_fetch;
extern crate linked_hash_map;
extern crate fetch;
extern crate parity_dapps_glue as parity_dapps;
extern crate futures;
extern crate parity_reactor;
#[macro_use]
extern crate log;
#[macro_use]
@ -81,6 +84,7 @@ mod rpc;
mod api;
mod proxypac;
mod url;
mod web;
#[cfg(test)]
mod tests;
@ -89,9 +93,11 @@ use std::net::SocketAddr;
use std::collections::HashMap;
use hash_fetch::urlhint::ContractClient;
use fetch::{Fetch, Client as FetchClient};
use jsonrpc_core::{IoHandler, IoDelegate};
use router::auth::{Authorization, NoAuth, HttpBasicAuth};
use ethcore_rpc::Extendable;
use parity_reactor::Remote;
use self::apps::{HOME_PAGE, DAPPS_DOMAIN};
@ -112,6 +118,8 @@ pub struct ServerBuilder {
registrar: Arc<ContractClient>,
sync_status: Arc<SyncStatus>,
signer_address: Option<(String, u16)>,
remote: Remote,
fetch: Option<FetchClient>,
}
impl Extendable for ServerBuilder {
@ -122,16 +130,23 @@ impl Extendable for ServerBuilder {
impl ServerBuilder {
/// Construct new dapps server
pub fn new(dapps_path: String, registrar: Arc<ContractClient>) -> Self {
pub fn new(dapps_path: String, registrar: Arc<ContractClient>, remote: Remote) -> Self {
ServerBuilder {
dapps_path: dapps_path,
handler: Arc::new(IoHandler::new()),
registrar: registrar,
sync_status: Arc::new(|| false),
signer_address: None,
remote: remote,
fetch: None,
}
}
/// Set a fetch client to use.
pub fn with_fetch(&mut self, fetch: FetchClient) {
self.fetch = Some(fetch);
}
/// Change default sync status.
pub fn with_sync_status(&mut self, status: Arc<SyncStatus>) {
self.sync_status = status;
@ -154,6 +169,8 @@ impl ServerBuilder {
self.signer_address.clone(),
self.registrar.clone(),
self.sync_status.clone(),
self.remote.clone(),
try!(self.fetch()),
)
}
@ -169,8 +186,17 @@ impl ServerBuilder {
self.signer_address.clone(),
self.registrar.clone(),
self.sync_status.clone(),
self.remote.clone(),
try!(self.fetch()),
)
}
fn fetch(&self) -> Result<FetchClient, ServerError> {
match self.fetch.clone() {
Some(fetch) => Ok(fetch),
None => FetchClient::new().map_err(|_| ServerError::FetchInitialization),
}
}
}
/// Webapps HTTP server.
@ -206,7 +232,7 @@ impl Server {
}
}
fn start_http<A: Authorization + 'static>(
fn start_http<A: Authorization + 'static, F: Fetch>(
addr: &SocketAddr,
hosts: Option<Vec<String>>,
authorization: A,
@ -215,11 +241,19 @@ impl Server {
signer_address: Option<(String, u16)>,
registrar: Arc<ContractClient>,
sync_status: Arc<SyncStatus>,
remote: Remote,
fetch: F,
) -> Result<Server, ServerError> {
let panic_handler = Arc::new(Mutex::new(None));
let authorization = Arc::new(authorization);
let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(hash_fetch::urlhint::URLHintContract::new(registrar), sync_status, signer_address.clone()));
let endpoints = Arc::new(apps::all_endpoints(dapps_path, signer_address.clone()));
let content_fetcher = Arc::new(apps::fetcher::ContentFetcher::new(
hash_fetch::urlhint::URLHintContract::new(registrar),
sync_status,
signer_address.clone(),
remote.clone(),
fetch.clone(),
));
let endpoints = Arc::new(apps::all_endpoints(dapps_path, signer_address.clone(), remote.clone(), fetch.clone()));
let cors_domains = Self::cors_domains(signer_address.clone());
let special = Arc::new({
@ -287,6 +321,8 @@ pub enum ServerError {
IoError(std::io::Error),
/// Other `hyper` error
Other(hyper::error::Error),
/// Fetch service initialization error
FetchInitialization,
}
impl From<hyper::error::Error> for ServerError {
@ -299,7 +335,7 @@ impl From<hyper::error::Error> for ServerError {
}
/// Random filename
pub fn random_filename() -> String {
fn random_filename() -> String {
use ::rand::Rng;
let mut rng = ::rand::OsRng::new().unwrap();
rng.gen_ascii_chars().take(12).collect()

View File

@ -20,11 +20,12 @@ use std::fs;
use std::path::{Path, PathBuf};
use page::handler::{self, PageCache, PageHandlerWaiting};
use endpoint::{Endpoint, EndpointInfo, EndpointPath, Handler};
use mime::Mime;
#[derive(Debug, Clone)]
pub struct LocalPageEndpoint {
path: PathBuf,
mime: Option<String>,
mime: Option<Mime>,
info: Option<EndpointInfo>,
cache: PageCache,
embeddable_on: Option<(String, u16)>,
@ -41,7 +42,7 @@ impl LocalPageEndpoint {
}
}
pub fn single_file(path: PathBuf, mime: String, cache: PageCache) -> Self {
pub fn single_file(path: PathBuf, mime: Mime, cache: PageCache) -> Self {
LocalPageEndpoint {
path: path,
mime: Some(mime),
@ -55,9 +56,9 @@ impl LocalPageEndpoint {
self.path.clone()
}
fn page_handler_with_mime(&self, path: EndpointPath, mime: &str) -> handler::PageHandler<LocalSingleFile> {
fn page_handler_with_mime(&self, path: EndpointPath, mime: &Mime) -> handler::PageHandler<LocalSingleFile> {
handler::PageHandler {
app: LocalSingleFile { path: self.path.clone(), mime: mime.into() },
app: LocalSingleFile { path: self.path.clone(), mime: format!("{}", mime) },
prefix: None,
path: path,
file: handler::ServedFile::new(None),

View File

@ -27,7 +27,7 @@ use url::{Url, Host};
use hyper::{self, server, Next, Encoder, Decoder, Control, StatusCode};
use hyper::net::HttpStream;
use apps::{self, DAPPS_DOMAIN};
use apps::fetcher::ContentFetcher;
use apps::fetcher::Fetcher;
use endpoint::{Endpoint, Endpoints, EndpointPath};
use handlers::{Redirection, extract_url, ContentHandler};
use self::auth::{Authorization, Authorized};
@ -45,7 +45,7 @@ pub struct Router<A: Authorization + 'static> {
control: Option<Control>,
signer_address: Option<(String, u16)>,
endpoints: Arc<Endpoints>,
fetch: Arc<ContentFetcher>,
fetch: Arc<Fetcher>,
special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
authorization: Arc<A>,
allowed_hosts: Option<Vec<String>>,
@ -169,7 +169,7 @@ impl<A: Authorization> Router<A> {
pub fn new(
control: Control,
signer_address: Option<(String, u16)>,
content_fetcher: Arc<ContentFetcher>,
content_fetcher: Arc<Fetcher>,
endpoints: Arc<Endpoints>,
special: Arc<HashMap<SpecialEndpoint, Box<Endpoint>>>,
authorization: Arc<A>,

View File

@ -25,6 +25,7 @@ use Server;
use hash_fetch::urlhint::ContractClient;
use util::{Bytes, Address, Mutex, ToPretty};
use devtools::http_client;
use parity_reactor::Remote;
const REGISTRAR: &'static str = "8e4e9b13d4b45cb0befc93c3061b1408f67316b2";
const URLHINT: &'static str = "deadbeefcafe0000000000000000000000000000";
@ -74,7 +75,7 @@ pub fn init_server(hosts: Option<Vec<String>>, is_syncing: bool) -> (Server, Arc
let registrar = Arc::new(FakeRegistrar::new());
let mut dapps_path = env::temp_dir();
dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading");
let mut builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar.clone());
let mut builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar.clone(), Remote::new_sync());
builder.with_sync_status(Arc::new(move || is_syncing));
builder.with_signer_address(Some(("127.0.0.1".into(), SIGNER_PORT)));
(
@ -88,7 +89,7 @@ pub fn serve_with_auth(user: &str, pass: &str) -> Server {
let registrar = Arc::new(FakeRegistrar::new());
let mut dapps_path = env::temp_dir();
dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading");
let mut builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar);
let mut builder = ServerBuilder::new(dapps_path.to_str().unwrap().into(), registrar.clone(), Remote::new_sync());
builder.with_signer_address(Some(("127.0.0.1".into(), SIGNER_PORT)));
builder.start_basic_auth_http(&"127.0.0.1:0".parse().unwrap(), None, user, pass).unwrap()
}

180
dapps/src/web.rs Normal file
View File

@ -0,0 +1,180 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Serving web-based content (proxying)
use endpoint::{Endpoint, Handler, EndpointPath};
use handlers::{ContentFetcherHandler, ContentHandler, ContentValidator, Redirection, extract_url};
use page::{LocalPageEndpoint};
use fetch::{self, Fetch};
use url::Url;
use hyper::{self, server, net, Next, Encoder, Decoder};
use hyper::status::StatusCode;
use parity_reactor::Remote;
use apps::WEB_PATH;
pub struct Web<F> {
remote: Remote,
fetch: F,
}
impl<F: Fetch> Web<F> {
pub fn boxed(remote: Remote, fetch: F) -> Box<Endpoint> {
Box::new(Web {
remote: remote,
fetch: fetch,
})
}
}
impl<F: Fetch> Endpoint for Web<F> {
fn to_async_handler(&self, path: EndpointPath, control: hyper::Control) -> Box<Handler> {
Box::new(WebHandler {
control: control,
state: State::Initial,
path: path,
remote: self.remote.clone(),
fetch: self.fetch.clone(),
})
}
}
pub struct WebInstaller;
impl ContentValidator for WebInstaller {
type Error = String;
fn validate_and_install(&self, _response: fetch::Response) -> Result<LocalPageEndpoint, String> {
// let path = unimplemented!();
// let mime = response.content_type().unwrap_or(mime!(Text/Html));
// Ok(LocalPageEndpoint::single_file(path, mime, PageCache::Enabled))
Err("unimplemented".into())
}
}
enum State<F: Fetch> {
Initial,
Error(ContentHandler),
Redirecting(Redirection),
Fetching(ContentFetcherHandler<WebInstaller, F>),
}
struct WebHandler<F: Fetch> {
control: hyper::Control,
state: State<F>,
path: EndpointPath,
remote: Remote,
fetch: F,
}
impl<F: Fetch> WebHandler<F> {
fn extract_target_url(url: Option<Url>) -> Result<String, State<F>> {
let path = match url {
Some(url) => url.path,
None => {
return Err(State::Error(
ContentHandler::error(StatusCode::BadRequest, "Invalid URL", "Couldn't parse URL", None, None)
));
}
};
// TODO [ToDr] Check if token supplied in URL is correct.
// Support domain based routing.
let idx = match path.get(0).map(|m| m.as_ref()) {
Some(WEB_PATH) => 1,
_ => 0,
};
// Validate protocol
let protocol = match path.get(idx).map(|a| a.as_str()) {
Some("http") => "http",
Some("https") => "https",
_ => {
return Err(State::Error(
ContentHandler::error(StatusCode::BadRequest, "Invalid Protocol", "Invalid protocol used", None, None)
));
}
};
// Redirect if address to main page does not end with /
if let None = path.get(idx + 2) {
return Err(State::Redirecting(
Redirection::new(&format!("/{}/", path.join("/")))
));
}
Ok(format!("{}://{}", protocol, path[2..].join("/")))
}
}
impl<F: Fetch> server::Handler<net::HttpStream> for WebHandler<F> {
fn on_request(&mut self, request: server::Request<net::HttpStream>) -> Next {
let url = extract_url(&request);
// First extract the URL (reject invalid URLs)
let target_url = match Self::extract_target_url(url) {
Ok(url) => url,
Err(error) => {
self.state = error;
return Next::write();
}
};
let (mut handler, _control) = ContentFetcherHandler::new(
target_url,
self.path.clone(),
self.control.clone(),
WebInstaller,
None,
self.remote.clone(),
self.fetch.clone(),
);
let res = handler.on_request(request);
self.state = State::Fetching(handler);
res
}
fn on_request_readable(&mut self, decoder: &mut Decoder<net::HttpStream>) -> Next {
match self.state {
State::Initial => Next::end(),
State::Error(ref mut handler) => handler.on_request_readable(decoder),
State::Redirecting(ref mut handler) => handler.on_request_readable(decoder),
State::Fetching(ref mut handler) => handler.on_request_readable(decoder),
}
}
fn on_response(&mut self, res: &mut server::Response) -> Next {
match self.state {
State::Initial => Next::end(),
State::Error(ref mut handler) => handler.on_response(res),
State::Redirecting(ref mut handler) => handler.on_response(res),
State::Fetching(ref mut handler) => handler.on_response(res),
}
}
fn on_response_writable(&mut self, encoder: &mut Encoder<net::HttpStream>) -> Next {
match self.state {
State::Initial => Next::end(),
State::Error(ref mut handler) => handler.on_response_writable(encoder),
State::Redirecting(ref mut handler) => handler.on_response_writable(encoder),
State::Fetching(ref mut handler) => handler.on_response_writable(encoder),
}
}
}

View File

@ -7,9 +7,13 @@ version = "1.5.0"
authors = ["Parity Technologies <admin@parity.io>"]
[dependencies]
log = "0.3"
rustc-serialize = "0.3"
ethabi = "0.2.2"
futures = "0.1"
log = "0.3"
mime = "0.2"
mime_guess = "1.6.1"
rand = "0.3"
rustc-serialize = "0.3"
fetch = { path = "../util/fetch" }
ethcore-util = { path = "../util" }
parity-reactor = { path = "../util/reactor" }

View File

@ -17,13 +17,15 @@
//! Hash-addressed content resolver & fetcher.
use std::{io, fs};
use std::io::Write;
use std::sync::Arc;
use std::path::PathBuf;
use util::{Mutex, H256, sha3};
use fetch::{Fetch, FetchError, Client as FetchClient};
use fetch::{Fetch, Response, Error as FetchError, Client as FetchClient};
use futures::Future;
use parity_reactor::Remote;
use urlhint::{ContractClient, URLHintContract, URLHint, URLHintResult};
use util::{H256, sha3};
/// API for fetching by hash.
pub trait HashFetch: Send + Sync + 'static {
@ -33,7 +35,7 @@ pub trait HashFetch: Send + Sync + 'static {
/// 2. `on_done` - callback function invoked when the content is ready (or there was error during fetch)
///
/// This function may fail immediately when fetch cannot be initialized or content cannot be resolved.
fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>) -> Result<(), Error>;
fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>);
}
/// Hash-fetching error.
@ -67,44 +69,61 @@ impl From<io::Error> for Error {
}
/// Default Hash-fetching client using on-chain contract to resolve hashes to URLs.
pub struct Client {
pub struct Client<F: Fetch + 'static = FetchClient> {
contract: URLHintContract,
fetch: Mutex<FetchClient>,
fetch: F,
remote: Remote,
}
impl Client {
/// Creates new instance of the `Client` given on-chain contract client.
pub fn new(contract: Arc<ContractClient>) -> Self {
/// Creates new instance of the `Client` given on-chain contract client and task runner.
pub fn new(contract: Arc<ContractClient>, remote: Remote) -> Self {
Client::with_fetch(contract, FetchClient::new().unwrap(), remote)
}
}
impl<F: Fetch + 'static> Client<F> {
/// Creates new instance of the `Client` given on-chain contract client, fetch service and task runner.
pub fn with_fetch(contract: Arc<ContractClient>, fetch: F, remote: Remote) -> Self {
Client {
contract: URLHintContract::new(contract),
fetch: Mutex::new(FetchClient::default()),
fetch: fetch,
remote: remote,
}
}
}
impl HashFetch for Client {
fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>) -> Result<(), Error> {
impl<F: Fetch + 'static> HashFetch for Client<F> {
fn fetch(&self, hash: H256, on_done: Box<Fn(Result<PathBuf, Error>) + Send>) {
debug!(target: "fetch", "Fetching: {:?}", hash);
let url = try!(
self.contract.resolve(hash.to_vec()).map(|content| match content {
let url = self.contract.resolve(hash.to_vec()).map(|content| match content {
URLHintResult::Dapp(dapp) => {
dapp.url()
},
URLHintResult::Content(content) => {
content.url
},
}).ok_or_else(|| Error::NoResolution)
);
}).ok_or_else(|| Error::NoResolution);
debug!(target: "fetch", "Resolved {:?} to {:?}. Fetching...", hash, url);
self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| {
fn validate_hash(hash: H256, result: Result<PathBuf, FetchError>) -> Result<PathBuf, Error> {
let path = try!(result);
match url {
Err(err) => on_done(Err(err)),
Ok(url) => {
let future = self.fetch.fetch(&url).then(move |result| {
fn validate_hash(path: PathBuf, hash: H256, result: Result<Response, FetchError>) -> Result<PathBuf, Error> {
let response = try!(result);
// Read the response
let mut reader = io::BufReader::new(response);
let mut writer = io::BufWriter::new(try!(fs::File::create(&path)));
try!(io::copy(&mut reader, &mut writer));
try!(writer.flush());
// And validate the hash
let mut file_reader = io::BufReader::new(try!(fs::File::open(&path)));
let content_hash = try!(sha3(&mut file_reader));
if content_hash != hash {
Err(Error::HashMismatch{ got: content_hash, expected: hash })
} else {
@ -113,7 +132,31 @@ impl HashFetch for Client {
}
debug!(target: "fetch", "Content fetched, validating hash ({:?})", hash);
on_done(validate_hash(hash, result))
})).map_err(Into::into)
let path = random_temp_path();
let res = validate_hash(path.clone(), hash, result);
if let Err(ref err) = res {
trace!(target: "fetch", "Error: {:?}", err);
// Remove temporary file in case of error
let _ = fs::remove_dir_all(&path);
}
on_done(res);
Ok(()) as Result<(), ()>
});
self.remote.spawn(self.fetch.process(future));
},
}
}
}
fn random_temp_path() -> PathBuf {
use ::rand::Rng;
use ::std::env;
let mut rng = ::rand::OsRng::new().expect("Reliable random source is required to work.");
let file: String = rng.gen_ascii_chars().take(12).collect();
let mut path = env::temp_dir();
path.push(file);
path
}

View File

@ -20,11 +20,17 @@
#[macro_use]
extern crate log;
extern crate rustc_serialize;
extern crate mime_guess;
#[macro_use]
extern crate mime;
extern crate ethabi;
extern crate ethcore_util as util;
extern crate fetch;
pub extern crate fetch;
extern crate futures;
extern crate mime_guess;
extern crate rand;
extern crate rustc_serialize;
extern crate parity_reactor;
mod client;

View File

@ -19,6 +19,7 @@
use std::fmt;
use std::sync::Arc;
use rustc_serialize::hex::ToHex;
use mime::Mime;
use mime_guess;
use ethabi::{Interface, Contract, Token};
@ -76,7 +77,7 @@ pub struct Content {
/// URL of the content
pub url: String,
/// MIME type of the content
pub mime: String,
pub mime: Mime,
/// Content owner address
pub owner: Address,
}
@ -183,7 +184,7 @@ impl URLHintContract {
let commit = GithubApp::commit(&commit);
if commit == Some(Default::default()) {
let mime = guess_mime_type(&account_slash_repo).unwrap_or("application/octet-stream".into());
let mime = guess_mime_type(&account_slash_repo).unwrap_or(mime!(Application/_));
return Some(URLHintResult::Content(Content {
url: account_slash_repo,
mime: mime,
@ -235,7 +236,7 @@ impl URLHint for URLHintContract {
}
}
fn guess_mime_type(url: &str) -> Option<String> {
fn guess_mime_type(url: &str) -> Option<Mime> {
const CONTENT_TYPE: &'static str = "content-type=";
let mut it = url.split('#');
@ -247,14 +248,14 @@ fn guess_mime_type(url: &str) -> Option<String> {
for meta in metas.split('&') {
let meta = meta.to_lowercase();
if meta.starts_with(CONTENT_TYPE) {
return Some(meta[CONTENT_TYPE.len()..].to_owned());
return meta[CONTENT_TYPE.len()..].parse().ok();
}
}
}
url.and_then(|url| {
url.split('.').last()
}).and_then(|extension| {
mime_guess::get_mime_type_str(extension).map(Into::into)
mime_guess::get_mime_type_opt(extension)
})
}
@ -369,7 +370,7 @@ mod tests {
// then
assert_eq!(res, Some(URLHintResult::Content(Content {
url: "https://ethcore.io/assets/images/ethcore-black-horizontal.png".into(),
mime: "image/png".into(),
mime: mime!(Image/Png),
owner: Address::from_str("deadcafebeefbeefcafedeaddeedfeedffffffff").unwrap(),
})))
}
@ -401,9 +402,9 @@ mod tests {
assert_eq!(guess_mime_type(url1), None);
assert_eq!(guess_mime_type(url2), Some("image/png".into()));
assert_eq!(guess_mime_type(url3), Some("image/png".into()));
assert_eq!(guess_mime_type(url4), Some("image/jpeg".into()));
assert_eq!(guess_mime_type(url5), Some("image/png".into()));
assert_eq!(guess_mime_type(url2), Some(mime!(Image/Png)));
assert_eq!(guess_mime_type(url3), Some(mime!(Image/Png)));
assert_eq!(guess_mime_type(url4), Some(mime!(Image/Jpeg)));
assert_eq!(guess_mime_type(url5), Some(mime!(Image/Png)));
}
}

View File

@ -21,6 +21,8 @@ use ethcore::client::Client;
use ethsync::SyncProvider;
use helpers::replace_home;
use dir::default_data_path;
use hash_fetch::fetch::Client as FetchClient;
use parity_reactor::Remote;
#[derive(Debug, PartialEq, Clone)]
pub struct Configuration {
@ -53,6 +55,8 @@ pub struct Dependencies {
pub apis: Arc<rpc_apis::Dependencies>,
pub client: Arc<Client>,
pub sync: Arc<SyncProvider>,
pub remote: Remote,
pub fetch: FetchClient,
}
pub fn new(configuration: Configuration, deps: Dependencies) -> Result<Option<WebappServer>, String> {
@ -128,10 +132,12 @@ mod server {
let mut server = dapps::ServerBuilder::new(
dapps_path,
Arc::new(Registrar { client: deps.client.clone() })
Arc::new(Registrar { client: deps.client.clone() }),
deps.remote.clone(),
);
let sync = deps.sync.clone();
let client = deps.client.clone();
server.with_fetch(deps.fetch.clone());
server.with_sync_status(Arc::new(move || is_major_importing(Some(sync.status().state), client.queue_info())));
server.with_signer_address(signer_address);

View File

@ -56,6 +56,7 @@ extern crate regex;
extern crate isatty;
extern crate toml;
extern crate app_dirs;
extern crate parity_reactor;
#[macro_use]
extern crate ethcore_util as util;

View File

@ -28,6 +28,8 @@ use ethsync::{ManageNetwork, SyncProvider};
use ethcore_rpc::{Extendable, NetworkSettings};
pub use ethcore_rpc::SignerService;
use updater::Updater;
use hash_fetch::fetch::Client as FetchClient;
use parity_reactor::Remote;
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub enum Api {
@ -122,6 +124,8 @@ pub struct Dependencies {
pub geth_compatibility: bool,
pub dapps_interface: Option<String>,
pub dapps_port: Option<u16>,
pub fetch: FetchClient,
pub remote: Remote,
}
fn to_modules(apis: &[Api]) -> BTreeMap<String, String> {
@ -242,7 +246,14 @@ pub fn setup_rpc<T: Extendable>(server: T, deps: Arc<Dependencies>, apis: ApiSet
server.add_delegate(ParityAccountsClient::new(&deps.secret_store, &deps.client).to_delegate());
},
Api::ParitySet => {
server.add_delegate(ParitySetClient::new(&deps.client, &deps.miner, &deps.updater, &deps.net_service).to_delegate())
server.add_delegate(ParitySetClient::new(
&deps.client,
&deps.miner,
&deps.updater,
&deps.net_service,
deps.fetch.clone(),
deps.remote.clone(),
).to_delegate())
},
Api::Traces => {
server.add_delegate(TracesClient::new(&deps.client, &deps.miner).to_delegate())

View File

@ -32,6 +32,8 @@ use ethcore::verification::queue::VerifierSettings;
use ethsync::SyncConfig;
use informant::Informant;
use updater::{UpdatePolicy, Updater};
use parity_reactor::{EventLoop, EventLoopHandle};
use hash_fetch::fetch::Client as FetchClient;
use rpc::{HttpServer, IpcServer, HttpConfiguration, IpcConfiguration};
use signer::SignerServer;
@ -304,10 +306,23 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
chain_notify.start();
}
// spin up event loop
let event_loop = EventLoop::spawn();
// fetch service
let fetch = try!(FetchClient::new().map_err(|e| format!("Error starting fetch client: {:?}", e)));
// the updater service
let updater = Updater::new(Arc::downgrade(&(service.client() as Arc<BlockChainClient>)), Arc::downgrade(&sync_provider), update_policy);
let updater = Updater::new(
Arc::downgrade(&(service.client() as Arc<BlockChainClient>)),
Arc::downgrade(&sync_provider),
update_policy,
fetch.clone(),
event_loop.remote(),
);
service.add_notify(updater.clone());
// set up dependencies for rpc servers
let signer_path = cmd.signer_conf.signer_path.clone();
let deps_for_rpc_apis = Arc::new(rpc_apis::Dependencies {
@ -334,6 +349,8 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
true => Some(cmd.dapps_conf.port),
false => None,
},
fetch: fetch.clone(),
remote: event_loop.remote(),
});
let dependencies = rpc::Dependencies {
@ -351,6 +368,8 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
apis: deps_for_rpc_apis.clone(),
client: client.clone(),
sync: sync_provider.clone(),
remote: event_loop.remote(),
fetch: fetch.clone(),
};
let dapps_server = try!(dapps::new(cmd.dapps_conf.clone(), dapps_deps));
@ -409,7 +428,16 @@ pub fn execute(cmd: RunCmd, can_restart: bool, logger: Arc<RotatingLogger>) -> R
}
// Handle exit
let restart = wait_for_exit(panic_handler, http_server, ipc_server, dapps_server, signer_server, updater, can_restart);
let restart = wait_for_exit(
panic_handler,
http_server,
ipc_server,
dapps_server,
signer_server,
event_loop.into(),
updater,
can_restart,
);
info!("Finishing work, please wait...");
@ -468,6 +496,7 @@ fn wait_for_exit(
_ipc_server: Option<IpcServer>,
_dapps_server: Option<WebappServer>,
_signer_server: Option<SignerServer>,
_event_loop: EventLoopHandle,
updater: Arc<Updater>,
can_restart: bool
) -> bool {

View File

@ -9,9 +9,15 @@ build = "build.rs"
[lib]
[dependencies]
futures = "0.1"
log = "0.3"
rustc-serialize = "0.3"
serde = "0.8"
serde_json = "0.8"
time = "0.1"
rand = "0.3"
semver = "0.5"
transient-hashmap = "0.1"
jsonrpc-core = { git = "https://github.com/ethcore/jsonrpc.git" }
jsonrpc-http-server = { git = "https://github.com/ethcore/jsonrpc.git" }
jsonrpc-ipc-server = { git = "https://github.com/ethcore/jsonrpc.git" }
@ -29,13 +35,10 @@ ethcore-devtools = { path = "../devtools" }
parity-updater = { path = "../updater" }
rlp = { path = "../util/rlp" }
fetch = { path = "../util/fetch" }
rustc-serialize = "0.3"
transient-hashmap = "0.1"
parity-reactor = { path = "../util/reactor" }
serde_macros = { version = "0.8.0", optional = true }
clippy = { version = "0.0.103", optional = true}
ethcore-ipc = { path = "../ipc/rpc" }
time = "0.1"
semver = "0.5"
[build-dependencies]
serde_codegen = { version = "0.8.0", optional = true }

View File

@ -38,7 +38,10 @@ extern crate ethcore_ipc;
extern crate time;
extern crate rlp;
extern crate fetch;
extern crate futures;
extern crate rand;
extern crate parity_updater as updater;
extern crate parity_reactor;
#[macro_use]
extern crate log;

View File

@ -24,7 +24,6 @@ use std::fmt;
use rlp::DecoderError;
use ethcore::error::{Error as EthcoreError, CallError, TransactionError};
use ethcore::account_provider::{Error as AccountError};
use fetch::FetchError;
use jsonrpc_core::{Error, ErrorCode, Value};
mod codes {
@ -203,7 +202,7 @@ pub fn encryption_error<T: fmt::Debug>(error: T) -> Error {
}
}
pub fn from_fetch_error(error: FetchError) -> Error {
pub fn from_fetch_error<T: fmt::Debug>(error: T) -> Error {
Error {
code: ErrorCode::ServerError(codes::FETCH_ERROR),
message: "Error while fetching content.".into(),

View File

@ -15,16 +15,18 @@
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
/// Parity-specific rpc interface for operations altering the settings.
use std::{fs, io};
use std::sync::{Arc, Weak, mpsc};
use std::io;
use std::sync::{Arc, Weak};
use ethcore::miner::MinerService;
use ethcore::client::MiningBlockChainClient;
use ethcore::mode::Mode;
use ethsync::ManageNetwork;
use fetch::{Client as FetchClient, Fetch};
use util::{Mutex, sha3};
use fetch::{self, Fetch};
use futures::Future;
use util::sha3;
use updater::{Service as UpdateService};
use parity_reactor::Remote;
use jsonrpc_core::Error;
use jsonrpc_macros::Ready;
@ -33,7 +35,7 @@ use v1::traits::ParitySet;
use v1::types::{Bytes, H160, H256, U256, ReleaseInfo};
/// Parity-specific rpc interface for operations altering the settings.
pub struct ParitySetClient<C, M, U, F=FetchClient> where
pub struct ParitySetClient<C, M, U, F=fetch::Client> where
C: MiningBlockChainClient,
M: MinerService,
U: UpdateService,
@ -43,18 +45,8 @@ pub struct ParitySetClient<C, M, U, F=FetchClient> where
miner: Weak<M>,
updater: Weak<U>,
net: Weak<ManageNetwork>,
fetch: Mutex<F>,
}
impl<C, M, U> ParitySetClient<C, M, U, FetchClient> where
C: MiningBlockChainClient,
M: MinerService,
U: UpdateService,
{
/// Creates new `ParitySetClient` with default `FetchClient`.
pub fn new(client: &Arc<C>, miner: &Arc<M>, updater: &Arc<U>, net: &Arc<ManageNetwork>) -> Self {
Self::with_fetch(client, miner, updater, net)
}
fetch: F,
remote: Remote,
}
impl<C, M, U, F> ParitySetClient<C, M, U, F> where
@ -63,14 +55,15 @@ impl<C, M, U, F> ParitySetClient<C, M, U, F> where
U: UpdateService,
F: Fetch,
{
/// Creates new `ParitySetClient` with default `FetchClient`.
pub fn with_fetch(client: &Arc<C>, miner: &Arc<M>, updater: &Arc<U>, net: &Arc<ManageNetwork>) -> Self {
/// Creates new `ParitySetClient` with given `Fetch`.
pub fn new(client: &Arc<C>, miner: &Arc<M>, updater: &Arc<U>, net: &Arc<ManageNetwork>, fetch: F, remote: Remote) -> Self {
ParitySetClient {
client: Arc::downgrade(client),
miner: Arc::downgrade(miner),
updater: Arc::downgrade(updater),
net: Arc::downgrade(net),
fetch: Mutex::new(F::default()),
fetch: fetch,
remote: remote,
}
}
@ -199,41 +192,23 @@ impl<C, M, U, F> ParitySet for ParitySetClient<C, M, U, F> where
fn hash_content(&self, ready: Ready<H256>, url: String) {
let res = self.active();
let hash_content = |result| {
let path = try!(result);
let mut file = io::BufReader::new(try!(fs::File::open(&path)));
// Try to hash
let result = sha3(&mut file);
// Remove file (always)
try!(fs::remove_file(&path));
// Return the result
Ok(try!(result))
};
match res {
Err(e) => ready.ready(Err(e)),
Ok(()) => {
let (tx, rx) = mpsc::channel();
let res = self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| {
let result = hash_content(result)
let task = self.fetch.fetch(&url).then(move |result| {
let result = result
.map_err(errors::from_fetch_error)
.and_then(|response| {
sha3(&mut io::BufReader::new(response)).map_err(errors::from_fetch_error)
})
.map(Into::into);
// Receive ready and invoke with result.
let ready: Ready<H256> = rx.recv().expect(
"recv() fails when `tx` has been dropped, if this closure is invoked `tx` is not dropped (`res == Ok()`); qed"
);
ready.ready(result);
}));
Ok(()) as Result<(), ()>
});
// Either invoke ready right away or transfer it to the closure.
if let Err(e) = res {
ready.ready(Err(errors::from_fetch_error(e)));
} else {
tx.send(ready).expect(
"send() fails when `rx` end is dropped, if `res == Ok()`: `rx` is moved to the closure; qed"
);
}
self.remote.spawn(task);
}
}
}

View File

@ -16,29 +16,24 @@
//! Test implementation of fetch client.
use std::io::Write;
use std::{env, fs, thread};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use fetch::{Fetch, FetchError, FetchResult};
use std::{io, thread};
use futures::{self, Future};
use fetch::{self, Fetch};
/// Test implementation of fetcher. Will always return the same file.
#[derive(Default)]
#[derive(Default, Clone)]
pub struct TestFetch;
impl Fetch for TestFetch {
fn request_async(&mut self, _url: &str, _abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError> {
type Result = futures::BoxFuture<fetch::Response, fetch::Error>;
fn fetch_with_abort(&self, _url: &str, _abort: fetch::Abort) -> Self::Result {
let (tx, rx) = futures::oneshot();
thread::spawn(move || {
let mut path = env::temp_dir();
path.push(Self::random_filename());
let mut file = fs::File::create(&path).unwrap();
file.write_all(b"Some content").unwrap();
on_done(Ok(path));
let cursor = io::Cursor::new(b"Some content");
tx.complete(fetch::Response::from_reader(cursor));
});
Ok(())
rx.map_err(|_| fetch::Error::Aborted).boxed()
}
}

View File

@ -22,6 +22,7 @@ use util::{U256, Address};
use ethcore::miner::MinerService;
use ethcore::client::TestBlockChainClient;
use ethsync::ManageNetwork;
use parity_reactor::Remote;
use jsonrpc_core::{IoHandler, GenericIoHandler};
use v1::{ParitySet, ParitySetClient};
@ -47,7 +48,7 @@ fn updater_service() -> Arc<TestUpdater> {
pub type TestParitySetClient = ParitySetClient<TestBlockChainClient, TestMinerService, TestUpdater, TestFetch>;
fn parity_set_client(client: &Arc<TestBlockChainClient>, miner: &Arc<TestMinerService>, updater: &Arc<TestUpdater>, net: &Arc<TestManageNetwork>) -> TestParitySetClient {
ParitySetClient::with_fetch(client, miner, updater, &(net.clone() as Arc<ManageNetwork>))
ParitySetClient::new(client, miner, updater, &(net.clone() as Arc<ManageNetwork>), TestFetch::default(), Remote::new_sync())
}
#[test]

View File

@ -12,13 +12,14 @@ ethcore-ipc-codegen = { path = "../ipc/codegen" }
[dependencies]
log = "0.3"
ethabi = "0.2.2"
target_info = "0.1"
ethcore = { path = "../ethcore" }
ethsync = { path = "../sync" }
ethcore-util = { path = "../util" }
parity-hash-fetch = { path = "../hash-fetch" }
ipc-common-types = { path = "../ipc-common-types" }
ethcore-ipc = { path = "../ipc/rpc" }
target_info = "0.1"
parity-reactor = { path = "../util/reactor" }
[profile.release]
debug = true

View File

@ -25,6 +25,7 @@ extern crate ethabi;
extern crate ethsync;
extern crate ethcore_ipc as ipc;
extern crate target_info;
extern crate parity_reactor;
mod updater;
mod operations;

View File

@ -26,7 +26,9 @@ use util::{Address, H160, H256, FixedHash, Mutex, Bytes};
use ethsync::{SyncProvider};
use ethcore::client::{BlockId, BlockChainClient, ChainNotify};
use hash_fetch::{self as fetch, HashFetch};
use hash_fetch::fetch::Client as FetchService;
use operations::Operations;
use parity_reactor::Remote;
use service::{Service};
use types::all::{ReleaseInfo, OperationsInfo, CapState};
@ -112,7 +114,7 @@ fn platform() -> String {
}
impl Updater {
pub fn new(client: Weak<BlockChainClient>, sync: Weak<SyncProvider>, update_policy: UpdatePolicy) -> Arc<Self> {
pub fn new(client: Weak<BlockChainClient>, sync: Weak<SyncProvider>, update_policy: UpdatePolicy, fetch: FetchService, remote: Remote) -> Arc<Self> {
let r = Arc::new(Updater {
update_policy: update_policy,
weak_self: Mutex::new(Default::default()),
@ -124,7 +126,7 @@ impl Updater {
this: VersionInfo::this(),
state: Mutex::new(Default::default()),
});
*r.fetcher.lock() = Some(fetch::Client::new(r.clone()));
*r.fetcher.lock() = Some(fetch::Client::with_fetch(r.clone(), fetch, remote));
*r.weak_self.lock() = Arc::downgrade(&r);
r.poll();
r
@ -273,7 +275,7 @@ impl Updater {
s.fetching = Some(latest.track.clone());
let weak_self = self.weak_self.lock().clone();
let f = move |r: Result<PathBuf, fetch::Error>| if let Some(this) = weak_self.upgrade() { this.fetch_done(r) };
self.fetcher.lock().as_ref().expect("Created on `new`; qed").fetch(b, Box::new(f)).ok();
self.fetcher.lock().as_ref().expect("Created on `new`; qed").fetch(b, Box::new(f));
}
}
}

View File

@ -7,11 +7,12 @@ version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
[dependencies]
futures = "0.1"
futures-cpupool = "0.1"
log = "0.3"
rand = "0.3"
hyper = { default-features = false, git = "https://github.com/ethcore/hyper" }
https-fetch = { path = "../https-fetch" }
clippy = { version = "0.0.90", optional = true }
reqwest = "0.2"
mime = "0.2"
clippy = { version = "0.0.90", optional = true}
[features]
default = []

View File

@ -16,131 +16,216 @@
//! Fetching
use std::{env, io};
use std::sync::{mpsc, Arc};
use std::sync::atomic::AtomicBool;
use std::path::PathBuf;
use std::{io, fmt};
use std::sync::Arc;
use std::sync::atomic::{self, AtomicBool};
use hyper;
use https_fetch as https;
use futures::{self, BoxFuture, Future};
use futures_cpupool::{CpuPool, CpuFuture};
use reqwest;
pub use mime::Mime;
use fetch_file::{FetchHandler, Error as HttpFetchError};
#[derive(Default, Debug, Clone)]
pub struct Abort(Arc<AtomicBool>);
pub type FetchResult = Result<PathBuf, FetchError>;
#[derive(Debug)]
pub enum FetchError {
InvalidUrl,
Http(HttpFetchError),
Https(https::FetchError),
Io(io::Error),
Other(String),
}
impl From<HttpFetchError> for FetchError {
fn from(e: HttpFetchError) -> Self {
FetchError::Http(e)
impl Abort {
pub fn is_aborted(&self) -> bool {
self.0.load(atomic::Ordering::SeqCst)
}
}
impl From<io::Error> for FetchError {
fn from(e: io::Error) -> Self {
FetchError::Io(e)
impl From<Arc<AtomicBool>> for Abort {
fn from(a: Arc<AtomicBool>) -> Self {
Abort(a)
}
}
pub trait Fetch: Default + Send {
/// Fetch URL and get the result in callback.
fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError>;
pub trait Fetch: Clone + Send + Sync + 'static {
type Result: Future<Item=Response, Error=Error> + Send + 'static;
/// Fetch URL and get a result Receiver. You will be notified when receiver is ready by `on_done` callback.
fn request(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn() + Send>) -> Result<mpsc::Receiver<FetchResult>, FetchError> {
let (tx, rx) = mpsc::channel();
try!(self.request_async(url, abort, Box::new(move |result| {
let res = tx.send(result);
if let Err(_) = res {
warn!("Fetch finished, but no one was listening");
/// Spawn the future in context of this `Fetch` thread pool.
/// Implementation is optional.
fn process<F>(&self, f: F) -> BoxFuture<(), ()> where
F: Future<Item=(), Error=()> + Send + 'static,
{
f.boxed()
}
on_done();
})));
Ok(rx)
/// Fetch URL and get a future for the result.
/// Supports aborting the request in the middle of execution.
fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result;
/// Fetch URL and get a future for the result.
fn fetch(&self, url: &str) -> Self::Result {
self.fetch_with_abort(url, Default::default())
}
/// Fetch URL and get the result synchronously.
fn fetch_sync(&self, url: &str) -> Result<Response, Error> {
self.fetch(url).wait()
}
/// Closes this client
fn close(self) {}
/// Returns a random filename
fn random_filename() -> String {
use ::rand::Rng;
let mut rng = ::rand::OsRng::new().unwrap();
rng.gen_ascii_chars().take(12).collect()
}
fn close(self) where Self: Sized {}
}
#[derive(Clone)]
pub struct Client {
http_client: hyper::Client<FetchHandler>,
https_client: https::Client,
client: Arc<reqwest::Client>,
pool: CpuPool,
limit: Option<usize>,
}
impl Default for Client {
fn default() -> Self {
// Max 15MB will be downloaded.
Client::with_limit(Some(15*1024*1024))
}
}
impl Client {
fn with_limit(limit: Option<usize>) -> Self {
Client {
http_client: hyper::Client::new().expect("Unable to initialize http client."),
https_client: https::Client::with_limit(limit).expect("Unable to initialize https client."),
pub fn new() -> Result<Self, Error> {
// Max 15MB will be downloaded.
Self::with_limit(Some(15*1024*1024))
}
fn with_limit(limit: Option<usize>) -> Result<Self, Error> {
let mut client = try!(reqwest::Client::new());
client.redirect(reqwest::RedirectPolicy::limited(5));
Ok(Client {
client: Arc::new(client),
pool: CpuPool::new(4),
limit: limit,
}
}
fn convert_url(url: hyper::Url) -> Result<https::Url, FetchError> {
let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl)));
let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl));
https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl)
}
fn temp_path() -> PathBuf {
let mut dir = env::temp_dir();
dir.push(Self::random_filename());
dir
})
}
}
impl Fetch for Client {
fn close(self) {
self.http_client.close();
self.https_client.close();
type Result = CpuFuture<Response, Error>;
fn process<F>(&self, f: F) -> BoxFuture<(), ()> where
F: Future<Item=(), Error=()> + Send + 'static,
{
self.pool.spawn(f).boxed()
}
fn request_async(&mut self, url: &str, abort: Arc<AtomicBool>, on_done: Box<Fn(FetchResult) + Send>) -> Result<(), FetchError> {
let is_https = url.starts_with("https://");
let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl));
let temp_path = Self::temp_path();
fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result {
debug!(target: "fetch", "Fetching from: {:?}", url);
trace!(target: "fetch", "Fetching from: {:?}", url);
if is_https {
let url = try!(Self::convert_url(url));
try!(self.https_client.fetch_to_file(
url,
temp_path.clone(),
abort,
move |result| on_done(result.map(|_| temp_path).map_err(FetchError::Https)),
).map_err(|e| FetchError::Other(format!("{:?}", e))));
} else {
try!(self.http_client.request(
url,
FetchHandler::new(temp_path, abort, Box::new(move |result| on_done(result)), self.limit.map(|v| v as u64).clone()),
).map_err(|e| FetchError::Other(format!("{:?}", e))));
}
Ok(())
self.pool.spawn(FetchTask {
url: url.into(),
client: self.client.clone(),
limit: self.limit,
abort: abort,
})
}
}
struct FetchTask {
url: String,
client: Arc<reqwest::Client>,
limit: Option<usize>,
abort: Abort,
}
impl Future for FetchTask {
// TODO [ToDr] timeouts handling?
type Item = Response;
type Error = Error;
fn poll(&mut self) -> futures::Poll<Self::Item, Self::Error> {
if self.abort.is_aborted() {
trace!(target: "fetch", "Fetch of {:?} aborted.", self.url);
return Err(Error::Aborted);
}
trace!(target: "fetch", "Starting fetch task: {:?}", self.url);
let result = try!(self.client.get(&self.url)
.header(reqwest::header::UserAgent("Parity Fetch".into()))
.send());
Ok(futures::Async::Ready(Response {
inner: ResponseInner::Response(result),
abort: self.abort.clone(),
limit: self.limit,
read: 0,
}))
}
}
#[derive(Debug)]
pub enum Error {
Fetch(reqwest::Error),
Aborted,
}
impl From<reqwest::Error> for Error {
fn from(error: reqwest::Error) -> Self {
Error::Fetch(error)
}
}
enum ResponseInner {
Response(reqwest::Response),
Reader(Box<io::Read + Send>),
}
impl fmt::Debug for ResponseInner {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
ResponseInner::Response(ref response) => response.fmt(f),
ResponseInner::Reader(_) => write!(f, "io Reader"),
}
}
}
#[derive(Debug)]
pub struct Response {
inner: ResponseInner,
abort: Abort,
limit: Option<usize>,
read: usize,
}
impl Response {
pub fn from_reader<R: io::Read + Send + 'static>(reader: R) -> Self {
Response {
inner: ResponseInner::Reader(Box::new(reader)),
abort: Abort::default(),
limit: None,
read: 0,
}
}
pub fn content_type(&self) -> Option<Mime> {
match self.inner {
ResponseInner::Response(ref r) => {
let content_type = r.headers().get::<reqwest::header::ContentType>();
content_type.map(|mime| mime.0.clone())
},
_ => None,
}
}
}
impl io::Read for Response {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.abort.is_aborted() {
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "Fetch aborted."));
}
let res = match self.inner {
ResponseInner::Response(ref mut response) => response.read(buf),
ResponseInner::Reader(ref mut reader) => reader.read(buf),
};
// increase bytes read
if let Ok(read) = res {
self.read += read;
}
// check limit
match self.limit {
Some(limit) if limit < self.read => {
return Err(io::Error::new(io::ErrorKind::PermissionDenied, "Size limit reached."));
},
_ => {},
}
res
}
}

View File

@ -1,176 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Hyper Client Handler to Fetch File
use std::{io, fs, fmt};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use hyper::status::StatusCode;
use hyper::client::{Request, Response, DefaultTransport as HttpStream};
use hyper::header::Connection;
use hyper::{self, Decoder, Encoder, Next};
use super::FetchError;
#[derive(Debug)]
pub enum Error {
Aborted,
NotStarted,
SizeLimit,
UnexpectedStatus(StatusCode),
IoError(io::Error),
HyperError(hyper::Error),
}
pub type FetchResult = Result<PathBuf, FetchError>;
pub type OnDone = Box<Fn(FetchResult) + Send>;
pub struct FetchHandler {
path: PathBuf,
abort: Arc<AtomicBool>,
file: Option<fs::File>,
result: Option<FetchResult>,
on_done: Option<OnDone>,
size_limit: Option<u64>,
}
impl fmt::Debug for FetchHandler {
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result)
}
}
impl Drop for FetchHandler {
fn drop(&mut self) {
let res = self.result.take().unwrap_or(Err(Error::NotStarted.into()));
// Remove file if there was an error
if res.is_err() || self.is_aborted() {
if let Some(file) = self.file.take() {
drop(file);
// Remove file
let _ = fs::remove_file(&self.path);
}
}
// send result
if let Some(f) = self.on_done.take() {
f(res);
}
}
}
impl FetchHandler {
pub fn new(path: PathBuf, abort: Arc<AtomicBool>, on_done: OnDone, size_limit: Option<u64>) -> Self {
FetchHandler {
path: path,
abort: abort,
file: None,
result: None,
on_done: Some(on_done),
size_limit: size_limit,
}
}
fn is_aborted(&self) -> bool {
self.abort.load(Ordering::SeqCst)
}
fn mark_aborted(&mut self) -> Next {
self.result = Some(Err(Error::Aborted.into()));
Next::end()
}
}
impl hyper::client::Handler<HttpStream> for FetchHandler {
fn on_request(&mut self, req: &mut Request) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
req.headers_mut().set(Connection::close());
read()
}
fn on_request_writable(&mut self, _encoder: &mut Encoder<HttpStream>) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
read()
}
fn on_response(&mut self, res: Response) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
if *res.status() != StatusCode::Ok {
self.result = Some(Err(Error::UnexpectedStatus(*res.status()).into()));
return Next::end();
}
// Open file to write
match fs::File::create(&self.path) {
Ok(file) => {
self.file = Some(file);
self.result = Some(Ok(self.path.clone()));
read()
},
Err(err) => {
self.result = Some(Err(Error::IoError(err).into()));
Next::end()
},
}
}
fn on_response_readable(&mut self, decoder: &mut Decoder<HttpStream>) -> Next {
if self.is_aborted() {
return self.mark_aborted();
}
match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) {
Ok(0) => Next::end(),
Ok(bytes_read) => match self.size_limit {
None => read(),
// Check limit
Some(limit) if limit > bytes_read => {
self.size_limit = Some(limit - bytes_read);
read()
},
// Size limit reached
_ => {
self.result = Some(Err(Error::SizeLimit.into()));
Next::end()
},
},
Err(e) => match e.kind() {
io::ErrorKind::WouldBlock => Next::read(),
_ => {
self.result = Some(Err(Error::IoError(e).into()));
Next::end()
}
}
}
}
fn on_error(&mut self, err: hyper::Error) -> Next {
self.result = Some(Err(Error::HyperError(err).into()));
Next::remove()
}
}
fn read() -> Next {
Next::read().timeout(Duration::from_secs(15))
}

View File

@ -18,12 +18,12 @@
#[macro_use]
extern crate log;
extern crate hyper;
extern crate https_fetch;
extern crate rand;
extern crate futures;
extern crate futures_cpupool;
extern crate mime;
extern crate reqwest;
pub mod client;
pub mod fetch_file;
pub use self::client::{Client, Fetch, FetchError, FetchResult};
pub use self::client::{Client, Fetch, Error, Response, Mime, Abort};

View File

@ -1,15 +0,0 @@
extern crate https_fetch;
use std::io;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use https_fetch::*;
fn main() {
let client = Client::new().unwrap();
let aborted = Arc::new(AtomicBool::new(false));
client.fetch(Url::new("github.com", 443, "/").unwrap(), Box::new(io::stdout()), aborted, |result| {
assert!(result.is_ok());
}).unwrap();
}

File diff suppressed because it is too large Load Diff

View File

@ -1,93 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs
MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL
MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug
RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm
+9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW
PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM
xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB
Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3
hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg
EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF
MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA
FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec
nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z
eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF
hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2
Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe
vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
+OkuE6N36B9K
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIEtjCCA56gAwIBAgIQDHmpRLCMEZUgkmFf4msdgzANBgkqhkiG9w0BAQsFADBs
MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j
ZSBFViBSb290IENBMB4XDTEzMTAyMjEyMDAwMFoXDTI4MTAyMjEyMDAwMFowdTEL
MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3
LmRpZ2ljZXJ0LmNvbTE0MDIGA1UEAxMrRGlnaUNlcnQgU0hBMiBFeHRlbmRlZCBW
YWxpZGF0aW9uIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
ggEBANdTpARR+JmmFkhLZyeqk0nQOe0MsLAAh/FnKIaFjI5j2ryxQDji0/XspQUY
uD0+xZkXMuwYjPrxDKZkIYXLBxA0sFKIKx9om9KxjxKws9LniB8f7zh3VFNfgHk/
LhqqqB5LKw2rt2O5Nbd9FLxZS99RStKh4gzikIKHaq7q12TWmFXo/a8aUGxUvBHy
/Urynbt/DvTVvo4WiRJV2MBxNO723C3sxIclho3YIeSwTQyJ3DkmF93215SF2AQh
cJ1vb/9cuhnhRctWVyh+HA1BV6q3uCe7seT6Ku8hI3UarS2bhjWMnHe1c63YlC3k
8wyd7sFOYn4XwHGeLN7x+RAoGTMCAwEAAaOCAUkwggFFMBIGA1UdEwEB/wQIMAYB
Af8CAQAwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
BQcDAjA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRp
Z2ljZXJ0LmNvbTBLBgNVHR8ERDBCMECgPqA8hjpodHRwOi8vY3JsNC5kaWdpY2Vy
dC5jb20vRGlnaUNlcnRIaWdoQXNzdXJhbmNlRVZSb290Q0EuY3JsMD0GA1UdIAQ2
MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5j
b20vQ1BTMB0GA1UdDgQWBBQ901Cl1qCt7vNKYApl0yHU+PjWDzAfBgNVHSMEGDAW
gBSxPsNpA/i/RwHUmCYaCALvY2QrwzANBgkqhkiG9w0BAQsFAAOCAQEAnbbQkIbh
hgLtxaDwNBx0wY12zIYKqPBKikLWP8ipTa18CK3mtlC4ohpNiAexKSHc59rGPCHg
4xFJcKx6HQGkyhE6V6t9VypAdP3THYUYUN9XR3WhfVUgLkc3UHKMf4Ib0mKPLQNa
2sPIoc4sUqIAY+tzunHISScjl2SFnjgOrWNoPLpSgVh5oywM395t6zHyuqB8bPEs
1OG9d4Q3A84ytciagRpKkk47RpqF/oOi+Z6Mo8wNXrM9zwR4jxQUezKcxwCmXMS1
oVWNWlZopCJwqjyBcdmdqEU79OX2olHdx3ti6G8MdOu42vi/hw15UJGQmxg7kVkn
8TUoE6smftX3eg==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIHeTCCBmGgAwIBAgIQC/20CQrXteZAwwsWyVKaJzANBgkqhkiG9w0BAQsFADB1
MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk
IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE2MDMxMDAwMDAwMFoXDTE4MDUxNzEy
MDAwMFowgf0xHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB
BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF
Ewc1MTU3NTUwMSQwIgYDVQQJExs4OCBDb2xpbiBQIEtlbGx5LCBKciBTdHJlZXQx
DjAMBgNVBBETBTk0MTA3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5p
YTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMu
MRMwEQYDVQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEA54hc8pZclxgcupjiA/F/OZGRwm/ZlucoQGTNTKmBEgNsrn/mxhngWmPw
bAvUaLP//T79Jc+1WXMpxMiz9PK6yZRRFuIo0d2bx423NA6hOL2RTtbnfs+y0PFS
/YTpQSelTuq+Fuwts5v6aAweNyMcYD0HBybkkdosFoDccBNzJ92Ac8I5EVDUc3Or
/4jSyZwzxu9kdmBlBzeHMvsqdH8SX9mNahXtXxRpwZnBiUjw36PgN+s9GLWGrafd
02T0ux9Yzd5ezkMxukqEAQ7AKIIijvaWPAJbK/52XLhIy2vpGNylyni/DQD18bBP
T+ZG1uv0QQP9LuY/joO+FKDOTler4wIDAQABo4IDejCCA3YwHwYDVR0jBBgwFoAU
PdNQpdagre7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFIhcSGcZzKB2WS0RecO+oqyH
IidbMCUGA1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1Ud
DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f
BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy
dmVyLWcxLmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt
ZXYtc2VydmVyLWcxLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsG
AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGI
BggrBgEFBQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0
LmNvbTBSBggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0Rp
Z2lDZXJ0U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMB
Af8EAjAAMIIBfwYKKwYBBAHWeQIEAgSCAW8EggFrAWkAdgCkuQmQtBhYFIe7E6LM
Z3AKPDWYBPkb37jjd80OyA3cEAAAAVNhieoeAAAEAwBHMEUCIQCHHSEY/ROK2/sO
ljbKaNEcKWz6BxHJNPOtjSyuVnSn4QIgJ6RqvYbSX1vKLeX7vpnOfCAfS2Y8lB5R
NMwk6us2QiAAdgBo9pj4H2SCvjqM7rkoHUz8cVFdZ5PURNEKZ6y7T0/7xAAAAVNh
iennAAAEAwBHMEUCIQDZpd5S+3to8k7lcDeWBhiJASiYTk2rNAT26lVaM3xhWwIg
NUqrkIODZpRg+khhp8ag65B8mu0p4JUAmkRDbiYnRvYAdwBWFAaaL9fC7NP14b1E
sj7HRna5vJkRXMDvlJhV1onQ3QAAAVNhieqZAAAEAwBIMEYCIQDnm3WStlvE99GC
izSx+UGtGmQk2WTokoPgo1hfiv8zIAIhAPrYeXrBgseA9jUWWoB4IvmcZtshjXso
nT8MIG1u1zF8MA0GCSqGSIb3DQEBCwUAA4IBAQCLbNtkxuspqycq8h1EpbmAX0wM
5DoW7hM/FVdz4LJ3Kmftyk1yd8j/PSxRrAQN2Mr/frKeK8NE1cMji32mJbBqpWtK
/+wC+avPplBUbNpzP53cuTMF/QssxItPGNP5/OT9Aj1BxA/NofWZKh4ufV7cz3pY
RDS4BF+EEFQ4l5GY+yp4WJA/xSvYsTHWeWxRD1/nl62/Rd9FN2NkacRVozCxRVle
FrBHTFxqIP6kDnxiLElBrZngtY07ietaYZVLQN/ETyqLQftsf8TecwTklbjvm8NT
JqbaIVifYwqwNN+4lRxS3F5lNlA/il12IOgbRioLI62o8G0DaEUQgHNf8vSG
-----END CERTIFICATE-----

View File

@ -1,220 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::cell::RefCell;
use std::{fs, str, thread};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::path::PathBuf;
use std::io::{self, Write};
use std::collections::HashMap;
use mio;
use tlsclient::{TlsClient, TlsClientError};
use url::Url;
#[derive(Debug)]
pub enum FetchError {
InvalidAddress,
ReadingCaCertificates,
UnexpectedStatus(String),
CaCertificates(io::Error),
Io(io::Error),
Notify(mio::NotifyError<ClientMessage>),
Client(TlsClientError),
}
impl From<io::Error> for FetchError {
fn from(e: io::Error) -> Self {
FetchError::Io(e)
}
}
impl From<mio::NotifyError<ClientMessage>> for FetchError {
fn from(e: mio::NotifyError<ClientMessage>) -> Self {
FetchError::Notify(e)
}
}
impl From<TlsClientError> for FetchError {
fn from(e: TlsClientError) -> Self {
FetchError::Client(e)
}
}
pub type FetchResult = Result<(), FetchError>;
pub enum ClientMessage {
Fetch(Url, Box<io::Write + Send>, Arc<AtomicBool>, Box<FnMut(FetchResult) + Send>),
Shutdown,
}
pub struct Client {
channel: mio::Sender<ClientMessage>,
thread: Option<thread::JoinHandle<()>>,
}
impl Drop for Client {
fn drop(&mut self) {
self.close_internal();
if let Some(thread) = self.thread.take() {
thread.join().expect("Clean shutdown.");
}
}
}
impl Client {
pub fn new() -> Result<Self, FetchError> {
Self::with_limit(None)
}
pub fn with_limit(size_limit: Option<usize>) -> Result<Self, FetchError> {
let mut event_loop = try!(mio::EventLoop::new());
let channel = event_loop.channel();
let thread = thread::spawn(move || {
let mut client = ClientLoop {
next_token: 0,
sessions: HashMap::new(),
size_limit: size_limit,
};
event_loop.run(&mut client).unwrap();
});
Ok(Client {
channel: channel,
thread: Some(thread),
})
}
pub fn fetch_to_file<F: FnOnce(FetchResult) + Send + 'static>(&self, url: Url, path: PathBuf, abort: Arc<AtomicBool>, callback: F) -> Result<(), FetchError> {
let file = try!(fs::File::create(&path));
self.fetch(url, Box::new(file), abort, move |result| {
if let Err(_) = result {
// remove temporary file
let _ = fs::remove_file(&path);
}
callback(result);
})
}
pub fn fetch<F: FnOnce(FetchResult) + Send + 'static>(&self, url: Url, writer: Box<io::Write + Send>, abort: Arc<AtomicBool>, callback: F) -> Result<(), FetchError> {
let cell = RefCell::new(Some(callback));
try!(self.channel.send(ClientMessage::Fetch(url, writer, abort, Box::new(move |res| {
cell.borrow_mut().take().expect("Called only once.")(res);
}))));
Ok(())
}
pub fn close(mut self) {
self.close_internal()
}
fn close_internal(&mut self) {
if let Err(e) = self.channel.send(ClientMessage::Shutdown) {
warn!("Error while closing client: {:?}. Already stopped?", e);
}
}
}
pub struct ClientLoop {
next_token: usize,
sessions: HashMap<usize, TlsClient>,
size_limit: Option<usize>,
}
impl mio::Handler for ClientLoop {
type Timeout = ();
type Message = ClientMessage;
fn ready(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>, token: mio::Token, events: mio::EventSet) {
let utoken = token.as_usize();
let remove = if let Some(mut tlsclient) = self.sessions.get_mut(&utoken) {
tlsclient.ready(event_loop, token, events)
} else {
false
};
if remove {
self.sessions.remove(&utoken);
}
}
fn notify(&mut self, event_loop: &mut mio::EventLoop<Self>, msg: Self::Message) {
match msg {
ClientMessage::Shutdown => event_loop.shutdown(),
ClientMessage::Fetch(url, writer, abort, callback) => {
let token = self.next_token;
self.next_token += 1;
if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback, self.size_limit.clone()) {
let httpreq = format!(
"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nUser-Agent: {}/{}\r\nAccept-Encoding: identity\r\n\r\n",
url.path(),
url.hostname(),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
);
debug!("Requesting content: {}", httpreq);
let _ = tlsclient.write(httpreq.as_bytes());
tlsclient.register(event_loop);
self.sessions.insert(token, tlsclient);
}
}
}
}
}
#[test]
fn should_successfuly_fetch_a_page() {
use std::io::{self, Cursor};
use std::sync::{mpsc, Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
struct Writer {
wrote: Arc<AtomicUsize>,
data: Cursor<Vec<u8>>,
}
impl io::Write for Writer {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let res = self.data.write(buf);
if let Ok(count) = res {
self.wrote.fetch_add(count, Ordering::Relaxed);
}
res
}
fn flush(&mut self) -> io::Result<()> { Ok(()) }
}
let client = Client::new().unwrap();
let wrote = Arc::new(AtomicUsize::new(0));
let writer = Writer {
wrote: wrote.clone(),
data: Cursor::new(Vec::new()),
};
let (tx, rx) = mpsc::channel();
client.fetch(Url::new("github.com", 443, "/").unwrap(), Box::new(writer), Arc::new(AtomicBool::new(false)), move |result| {
assert!(result.is_ok());
assert!(wrote.load(Ordering::Relaxed) > 0);
tx.send(result).unwrap();
}).unwrap();
let _ = rx.recv().unwrap();
}

View File

@ -1,399 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! HTTP format processor
use std::io::{self, Cursor, Write};
use std::cmp;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum State {
WaitingForStatus,
WaitingForHeaders,
WaitingForChunk,
WritingBody,
WritingChunk(usize),
Finished,
}
pub struct HttpProcessor {
state: State,
buffer: Cursor<Vec<u8>>,
status: Option<String>,
headers: Vec<String>,
body_writer: io::BufWriter<Box<io::Write>>,
size_limit: Option<usize>,
}
const BREAK_LEN: usize = 2;
impl HttpProcessor {
pub fn new(body_writer: Box<io::Write>, size_limit: Option<usize>) -> Self {
HttpProcessor {
state: State::WaitingForStatus,
buffer: Cursor::new(Vec::new()),
status: None,
headers: Vec::new(),
body_writer: io::BufWriter::new(body_writer),
size_limit: size_limit,
}
}
pub fn status(&self) -> Option<&String> {
self.status.as_ref()
}
pub fn status_is_ok(&self) -> bool {
self.status == Some("HTTP/1.1 200 OK".into())
}
#[cfg(test)]
pub fn headers(&self) -> &[String] {
&self.headers
}
fn find_break_index(&mut self) -> Option<usize> {
let data = self.buffer.get_ref();
let mut idx = 0;
let mut got_r = false;
// looks for \r\n in data
for b in data {
idx += 1;
if got_r && b == &10u8 {
return Some(idx);
} else if !got_r && b == &13u8 {
got_r = true;
} else {
got_r = false;
}
}
None
}
// Consumes bytes from internal buffer
fn buffer_consume(&mut self, bytes: usize) {
let bytes = cmp::min(bytes, self.buffer.get_ref().len());
// Drain data
self.buffer.get_mut().drain(0..bytes);
let len = self.buffer.position();
self.buffer.set_position(len - bytes as u64);
}
fn buffer_to_string(&mut self, bytes: usize) -> String {
let val = String::from_utf8_lossy(&self.buffer.get_ref()[0..bytes-BREAK_LEN]).into_owned();
self.buffer_consume(bytes);
val
}
fn is_chunked(&self) -> bool {
self.headers
.iter()
.find(|item| item.to_lowercase().contains("transfer-encoding: chunked"))
.is_some()
}
fn set_state(&mut self, state: State) {
self.state = state;
trace!("Changing state to {:?}", state);
}
fn process_buffer(&mut self) -> io::Result<()> {
// consume data and perform state transitions
loop {
match self.state {
State::WaitingForStatus => {
if let Some(break_index) = self.find_break_index() {
let status = self.buffer_to_string(break_index);
debug!("Read status: {:?}", status);
self.status = Some(status);
self.set_state(State::WaitingForHeaders);
} else {
// wait for more data
return Ok(());
}
},
State::WaitingForHeaders => {
match self.find_break_index() {
// Last header - got empty line, body starts
Some(BREAK_LEN) => {
self.buffer_consume(BREAK_LEN);
let is_chunked = self.is_chunked();
self.set_state(match is_chunked {
true => State::WaitingForChunk,
false => State::WritingBody,
});
},
Some(break_index) => {
let header = self.buffer_to_string(break_index);
debug!("Found header: {:?}", header);
self.headers.push(header);
},
None => return Ok(()),
}
},
State::WritingBody => {
let len = self.buffer.get_ref().len();
match self.size_limit {
None => {},
Some(limit) if limit > len => {},
_ => {
warn!("Finishing file fetching because limit was reached.");
self.set_state(State::Finished);
continue;
}
}
try!(self.body_writer.write_all(self.buffer.get_ref()));
self.buffer_consume(len);
return self.body_writer.flush();
},
State::WaitingForChunk => {
match self.find_break_index() {
None => return Ok(()),
// last chunk - size 0
Some(BREAK_LEN) => {
self.state = State::Finished;
},
Some(break_index) => {
let chunk_size = self.buffer_to_string(break_index);
self.set_state(if let Ok(size) = usize::from_str_radix(&chunk_size, 16) {
State::WritingChunk(size)
} else {
warn!("Error parsing server chunked response. Invalid chunk size.");
State::Finished
});
}
}
},
State::WritingChunk(0) => {
self.set_state(State::Finished);
},
// Buffers the data until we have a full chunk
State::WritingChunk(left) if self.buffer.get_ref().len() >= left => {
match self.size_limit {
None => {},
Some(limit) if limit > left => {
self.size_limit = Some(limit - left);
},
_ => {
warn!("Finishing file fetching because limit was reached.");
self.set_state(State::Finished);
continue;
}
}
{
let chunk = &self.buffer.get_ref()[0..left];
trace!("Writing chunk: {:?}", String::from_utf8_lossy(chunk));
try!(self.body_writer.write_all(chunk));
}
self.buffer_consume(left + BREAK_LEN);
self.set_state(State::WaitingForChunk);
},
// Wait for more data
State::WritingChunk(_) => return Ok(()),
// Just consume buffer
State::Finished => {
let len = self.buffer.get_ref().len();
self.buffer_consume(len);
return self.body_writer.flush();
},
}
}
}
#[cfg(test)]
pub fn state(&self) -> State {
self.state
}
}
impl io::Write for HttpProcessor {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let result = self.buffer.write(bytes);
try!(self.process_buffer());
result
}
fn flush(&mut self) -> io::Result<()> {
self.buffer.flush().and_then(|_| {
self.body_writer.flush()
})
}
}
#[cfg(test)]
mod tests {
use std::rc::Rc;
use std::cell::RefCell;
use std::io::{self, Write, Cursor};
use super::*;
struct Writer {
data: Rc<RefCell<Cursor<Vec<u8>>>>,
}
impl Writer {
fn new() -> (Self, Rc<RefCell<Cursor<Vec<u8>>>>) {
let data = Rc::new(RefCell::new(Cursor::new(Vec::new())));
(Writer { data: data.clone() }, data)
}
}
impl Write for Writer {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> { self.data.borrow_mut().write(buf) }
fn flush(&mut self) -> io::Result<()> { self.data.borrow_mut().flush() }
}
#[test]
fn should_be_able_to_process_status_line() {
// given
let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
// when
let out =
"\
HTTP/1.1 200 OK\r\n\
Server: Pari
";
http.write_all(out.as_bytes()).unwrap();
http.flush().unwrap();
// then
assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
assert_eq!(http.state(), State::WaitingForHeaders);
}
#[test]
fn should_be_able_to_process_headers() {
// given
let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None);
// when
let out =
"\
HTTP/1.1 200 OK\r\n\
Server: Parity/1.1.1\r\n\
Connection: close\r\n\
Content-Length: 2\r\n\
Content-Type: application/json\r\n\
\r\n\
";
http.write_all(out.as_bytes()).unwrap();
http.flush().unwrap();
// then
assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
assert_eq!(http.headers().len(), 4);
assert_eq!(http.state(), State::WritingBody);
}
#[test]
fn should_be_able_to_consume_body() {
// given
let (writer, data) = Writer::new();
let mut http = HttpProcessor::new(Box::new(writer), None);
// when
let out =
"\
HTTP/1.1 200 OK\r\n\
Server: Parity/1.1.1\r\n\
Connection: close\r\n\
Content-Length: 2\r\n\
Content-Type: application/json\r\n\
\r\n\
Some data\
";
http.write_all(out.as_bytes()).unwrap();
http.flush().unwrap();
// then
assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
assert_eq!(http.headers().len(), 4);
assert_eq!(http.state(), State::WritingBody);
assert_eq!(data.borrow().get_ref()[..], b"Some data"[..]);
}
#[test]
fn should_correctly_handle_chunked_content() {
// given
let (writer, data) = Writer::new();
let mut http = HttpProcessor::new(Box::new(writer), None);
// when
let out =
"\
HTTP/1.1 200 OK\r\n\
Host: 127.0.0.1:8080\r\n\
Transfer-Encoding: chunked\r\n\
Connection: close\r\n\
\r\n\
4\r\n\
Pari\r\n\
3\r\n\
ty \r\n\
D\r\n\
in\r\n\
\r\n\
chunks.\r\n\
0\r\n\
\r\n\
";
http.write_all(out.as_bytes()).unwrap();
http.flush().unwrap();
// then
assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
assert_eq!(http.headers().len(), 3);
assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]);
assert_eq!(http.state(), State::Finished);
}
#[test]
fn should_stop_fetching_when_limit_is_reached() {
// given
let (writer, data) = Writer::new();
let mut http = HttpProcessor::new(Box::new(writer), Some(5));
// when
let out =
"\
HTTP/1.1 200 OK\r\n\
Host: 127.0.0.1:8080\r\n\
Transfer-Encoding: chunked\r\n\
Connection: close\r\n\
\r\n\
4\r\n\
Pari\r\n\
3\r\n\
ty \r\n\
D\r\n\
in\r\n\
\r\n\
chunks.\r\n\
0\r\n\
\r\n\
";
http.write_all(out.as_bytes()).unwrap();
http.flush().unwrap();
// then
assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK");
assert_eq!(http.headers().len(), 3);
assert_eq!(data.borrow().get_ref()[..], b"Pari"[..]);
assert_eq!(http.state(), State::Finished);
}
}

View File

@ -1,28 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
extern crate rustls;
extern crate mio;
#[macro_use] extern crate log;
mod tlsclient;
mod client;
mod url;
mod http;
pub use self::client::{Client, FetchError, FetchResult};
pub use self::url::{Url, UrlError};

View File

@ -1,251 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::str;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::io::{self, Write, Read, Cursor, BufReader};
use mio;
use mio::tcp::TcpStream;
use rustls::{self, Session};
use url::Url;
use http::HttpProcessor;
use client::{FetchError, ClientLoop, FetchResult};
#[derive(Debug)]
pub enum TlsClientError {
Aborted,
Initialization,
UnexpectedEof,
Connection(io::Error),
Writer(io::Error),
Tls(rustls::TLSError),
}
/// This encapsulates the TCP-level connection, some connection
/// state, and the underlying TLS-level session.
pub struct TlsClient {
abort: Arc<AtomicBool>,
token: mio::Token,
socket: TcpStream,
tls_session: rustls::ClientSession,
writer: HttpProcessor,
error: Option<TlsClientError>,
closing: bool,
callback: Box<FnMut(FetchResult) + Send>,
}
impl io::Write for TlsClient {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.tls_session.write(bytes)
}
fn flush(&mut self) -> io::Result<()> {
self.tls_session.flush()
}
}
impl io::Read for TlsClient {
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
self.tls_session.read(bytes)
}
}
#[cfg(feature = "ca-github-only")]
static CA_CERTS: &'static [u8] = include_bytes!("./ca-github.crt");
#[cfg(not(feature = "ca-github-only"))]
static CA_CERTS: &'static [u8] = include_bytes!("./ca-certificates.crt");
impl TlsClient {
pub fn make_config() -> Result<Arc<rustls::ClientConfig>, FetchError> {
let mut config = rustls::ClientConfig::new();
let mut cursor = Cursor::new(CA_CERTS.to_vec());
let mut reader = BufReader::new(&mut cursor);
try!(config.root_store.add_pem_file(&mut reader).map_err(|_| FetchError::ReadingCaCertificates));
// TODO [ToDr] client certificate?
Ok(Arc::new(config))
}
pub fn new(
token: mio::Token,
url: &Url,
writer: Box<io::Write + Send>,
abort: Arc<AtomicBool>,
mut callback: Box<FnMut(FetchResult) + Send>,
size_limit: Option<usize>,
) -> Result<Self, FetchError> {
let res = TlsClient::make_config().and_then(|cfg| {
TcpStream::connect(url.address()).map(|sock| {
(cfg, sock)
}).map_err(Into::into)
});
match res {
Ok((cfg, sock)) => Ok(TlsClient {
abort: abort,
token: token,
writer: HttpProcessor::new(writer, size_limit),
socket: sock,
closing: false,
error: None,
tls_session: rustls::ClientSession::new(&cfg, url.hostname()),
callback: callback,
}),
Err(e) => {
callback(Err(e));
Err(FetchError::Client(TlsClientError::Initialization))
}
}
}
/// Called by mio each time events we register() for happen.
/// Return false if reregistering again.
pub fn ready(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>, token: mio::Token, events: mio::EventSet) -> bool {
assert_eq!(token, self.token);
let aborted = self.is_aborted();
if aborted {
// do_write needs to be invoked after that
self.tls_session.send_close_notify();
self.error = Some(TlsClientError::Aborted);
}
if events.is_readable() {
self.do_read();
}
if events.is_writable() {
self.do_write();
}
if self.is_closed() || aborted {
trace!("Connection closed");
let callback = &mut self.callback;
callback(match self.error.take() {
None if self.writer.status_is_ok() => Ok(()),
Some(err) => Err(err.into()),
None => {
Err(FetchError::UnexpectedStatus(format!("{:?}", self.writer.status())))
},
});
return true;
}
self.reregister(event_loop);
false
}
pub fn register(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>) {
event_loop.register(
&self.socket,
self.token,
self.event_set(),
mio::PollOpt::level() | mio::PollOpt::oneshot()
).unwrap_or_else(|e| self.error = Some(TlsClientError::Connection(e)));
}
fn reregister(&mut self, event_loop: &mut mio::EventLoop<ClientLoop>) {
event_loop.reregister(
&self.socket,
self.token,
self.event_set(),
mio::PollOpt::level() | mio::PollOpt::oneshot()
).unwrap_or_else(|e| self.error = Some(TlsClientError::Connection(e)));
}
/// We're ready to do a read.
fn do_read(&mut self) {
// Read TLS data. This fails if the underlying TCP connection is broken.
let rc = self.tls_session.read_tls(&mut self.socket);
if let Err(e) = rc {
trace!("TLS read error: {:?}", e);
self.closing = true;
self.error = Some(TlsClientError::Connection(e));
return;
}
// If we're ready but there's no data: EOF.
if rc.unwrap() == 0 {
trace!("Unexpected EOF");
self.error = Some(TlsClientError::UnexpectedEof);
self.closing = true;
return;
}
// Reading some TLS data might have yielded new TLS messages to process.
// Errors from this indicate TLS protocol problems and are fatal.
let processed = self.tls_session.process_new_packets();
if let Err(e) = processed {
trace!("TLS error: {:?}", e);
self.error = Some(TlsClientError::Tls(e));
self.closing = true;
return;
}
// Having read some TLS data, and processed any new messages, we might have new plaintext as a result.
// Read it and then write it to stdout.
let mut plaintext = Vec::new();
let rc = self.tls_session.read_to_end(&mut plaintext);
if !plaintext.is_empty() {
self.writer.write(&plaintext).unwrap_or_else(|e| {
trace!("Write error: {:?}", e);
self.error = Some(TlsClientError::Writer(e));
0
});
}
// If that fails, the peer might have started a clean TLS-level session closure.
if let Err(err) = rc {
if err.kind() != io::ErrorKind::ConnectionAborted {
self.error = Some(TlsClientError::Connection(err));
}
self.closing = true;
}
}
fn do_write(&mut self) {
self.tls_session.write_tls(&mut self.socket).unwrap_or_else(|e| {
warn!("TLS write error: {:?}", e);
0
});
}
// Use wants_read/wants_write to register for different mio-level IO readiness events.
fn event_set(&self) -> mio::EventSet {
let rd = self.tls_session.wants_read();
let wr = self.tls_session.wants_write();
if rd && wr {
mio::EventSet::readable() | mio::EventSet::writable()
} else if wr {
mio::EventSet::writable()
} else {
mio::EventSet::readable()
}
}
fn is_closed(&self) -> bool {
self.closing
}
fn is_aborted(&self) -> bool {
self.abort.load(Ordering::Relaxed)
}
}

View File

@ -1,81 +0,0 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
use std::net::SocketAddr;
#[derive(Debug)]
pub enum UrlError {
InvalidAddress
}
/// Build a ClientConfig from our arguments
pub struct Url {
address: SocketAddr,
hostname: String,
port: u16,
path: String,
}
impl Url {
pub fn new(hostname: &str, port: u16, path: &str) -> Result<Self, UrlError> {
let addr = try!(Self::lookup_ipv4(hostname, port));
Ok(Url {
address: addr,
hostname: hostname.into(),
port: port,
path: path.into(),
})
}
fn lookup_ipv4(host: &str, port: u16) -> Result<SocketAddr, UrlError> {
use std::net::ToSocketAddrs;
let addrs = try!((host, port).to_socket_addrs().map_err(|_| UrlError::InvalidAddress));
for addr in addrs {
if let SocketAddr::V4(_) = addr {
return Ok(addr.clone());
}
}
Err(UrlError::InvalidAddress)
}
pub fn address(&self) -> &SocketAddr {
&self.address
}
pub fn hostname(&self) -> &str {
&self.hostname
}
pub fn port(&self) -> u16 {
self.port
}
pub fn path(&self) -> &str {
&self.path
}
}
#[cfg(test)]
#[test]
fn should_parse_url() {
// given
let url = Url::new("github.com", 443, "/").unwrap();
assert_eq!(url.hostname(), "github.com");
assert_eq!(url.port(), 443);
assert_eq!(url.path(), "/");
}

11
util/reactor/Cargo.toml Normal file
View File

@ -0,0 +1,11 @@
[package]
description = "Parity Reactor"
homepage = "http://parity.io"
license = "GPL-3.0"
name = "parity-reactor"
version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
[dependencies]
futures = "0.1"
tokio-core = "0.1"

142
util/reactor/src/lib.rs Normal file
View File

@ -0,0 +1,142 @@
// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
// This file is part of Parity.
// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
//! Tokio Core Reactor wrapper.
extern crate futures;
extern crate tokio_core;
use std::thread;
use std::sync::mpsc;
use futures::{Future, IntoFuture};
use self::tokio_core::reactor::Remote as TokioRemote;
/// Event Loop for futures.
/// Wrapper around `tokio::reactor::Core`.
/// Runs in a separate thread.
pub struct EventLoop {
remote: Remote,
handle: EventLoopHandle,
}
impl EventLoop {
/// Spawns a new thread with `EventLoop` with given handler.
pub fn spawn() -> Self {
let (stop, stopped) = futures::oneshot();
let (tx, rx) = mpsc::channel();
let handle = thread::spawn(move || {
let mut el = tokio_core::reactor::Core::new().expect("Creating an event loop should not fail.");
tx.send(el.remote()).expect("Rx is blocking upper thread.");
let _ = el.run(futures::empty().select(stopped));
});
let remote = rx.recv().expect("tx is transfered to a newly spawned thread.");
EventLoop {
remote: Remote{
inner: Mode::Tokio(remote),
},
handle: EventLoopHandle {
close: Some(stop),
handle: Some(handle),
},
}
}
/// Returns event loop remote.
pub fn remote(&self) -> Remote {
self.remote.clone()
}
}
#[derive(Clone)]
enum Mode {
Tokio(TokioRemote),
Sync,
}
#[derive(Clone)]
pub struct Remote {
inner: Mode,
}
impl Remote {
/// Synchronous remote, used mostly for tests.
pub fn new_sync() -> Self {
Remote {
inner: Mode::Sync,
}
}
/// Spawn a future to this event loop
pub fn spawn<R>(&self, r: R) where
R: IntoFuture<Item=(), Error=()> + Send + 'static,
R::Future: 'static,
{
match self.inner {
Mode::Tokio(ref remote) => remote.spawn(move |_| r),
Mode::Sync => {
let _= r.into_future().wait();
},
}
}
/// Spawn a new future returned by given closure.
pub fn spawn_fn<F, R>(&self, f: F) where
F: FnOnce() -> R + Send + 'static,
R: IntoFuture<Item=(), Error=()>,
R::Future: 'static,
{
match self.inner {
Mode::Tokio(ref remote) => remote.spawn(move |_| f()),
Mode::Sync => {
let _ = f().into_future().wait();
},
}
}
}
/// A handle to running event loop. Dropping the handle will cause event loop to finish.
pub struct EventLoopHandle {
close: Option<futures::Complete<()>>,
handle: Option<thread::JoinHandle<()>>
}
impl From<EventLoop> for EventLoopHandle {
fn from(el: EventLoop) -> Self {
el.handle
}
}
impl Drop for EventLoopHandle {
fn drop(&mut self) {
self.close.take().map(|v| v.complete(()));
}
}
impl EventLoopHandle {
/// Blocks current thread and waits until the event loop is finished.
pub fn wait(mut self) -> thread::Result<()> {
self.handle.take()
.expect("Handle is taken only in `wait`, `wait` is consuming; qed").join()
}
/// Finishes this event loop.
pub fn close(mut self) {
self.close.take()
.expect("Close is taken only in `close` and `drop`. `close` is consuming; qed").complete(())
}
}