diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58036eb88..6c3b5389b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -266,6 +266,7 @@ test-linux: before_script: - git submodule update --init --recursive script: + - export RUST_BACKTRACE=1 - ./test.sh --verbose tags: - rust-test diff --git a/Cargo.lock b/Cargo.lock index 52bdee494..840684b2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,7 @@ dependencies = [ "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lru-cache 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -317,7 +318,7 @@ dependencies = [ "ethcore-devtools 1.4.0", "ethcore-rpc 1.4.0", "ethcore-util 1.4.0", - "https-fetch 0.1.0", + "fetch 0.1.0", "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)", @@ -466,6 +467,7 @@ dependencies = [ "ethkey 0.2.0", "ethstore 0.1.0", "ethsync 1.4.0", + "fetch 0.1.0", "json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)", "jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)", @@ -639,6 +641,16 @@ dependencies = [ "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fetch" +version = "0.1.0" +dependencies = [ + "https-fetch 0.1.0", + "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flate2" version = "0.2.14" @@ -690,7 +702,7 @@ version = "0.1.0" dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.5.1 (git+https://github.com/ethcore/mio?branch=v0.5.x)", - "rustls 0.1.1 (git+https://github.com/ctz/rustls)", + "rustls 0.1.2 (git+https://github.com/ctz/rustls)", ] [[package]] @@ -852,6 +864,11 @@ name = "libc" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "linked-hash-map" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -862,6 +879,14 @@ name = "log" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lru-cache" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.0.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "matches" version = "0.1.2" @@ -1345,7 +1370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ring" -version = "0.3.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1429,15 +1454,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.1.1" -source = "git+https://github.com/ctz/rustls#a9c5a79f49337e22ac05bb1ea114240bdbe0fdd2" +version = "0.1.2" +source = "git+https://github.com/ctz/rustls#3d2db624997004b7b18ba4463d6081f37598b2f5" dependencies = [ "base64 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1788,10 +1813,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "webpki" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1903,14 +1928,16 @@ dependencies = [ "checksum itoa 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1" "checksum json-ipc-server 0.2.4 (git+https://github.com/ethcore/json-ipc-server.git)" = "" "checksum json-tcp-server 0.1.0 (git+https://github.com/ethcore/json-tcp-server)" = "" -"checksum jsonrpc-core 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e913b3c809aab9378889da8b990b4a46b98bd4794c8117946a1cf63c5f87bcde" +"checksum jsonrpc-core 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3c5094610b07f28f3edaf3947b732dadb31dbba4941d4d0c1c7a8350208f4414" "checksum jsonrpc-http-server 6.1.0 (git+https://github.com/ethcore/jsonrpc-http-server.git)" = "" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f" "checksum libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "23e3757828fa702a20072c37ff47938e9dd331b92fac6e223d26d4b7a55f7ee2" +"checksum linked-hash-map 0.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "83f7ff3baae999fdf921cccf54b61842bb3b26868d50d02dff48052ebec8dd79" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" +"checksum lru-cache 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "42d50dcb5d9f145df83b1043207e1ac0c37c9c779c4e128ca4655abc3f3cbf8c" "checksum matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "15305656809ce5a4805b1ff2946892810992197ce1270ff79baded852187942e" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a74cc2587bf97c49f3f5bab62860d6abf3902ca73b66b51d9b049fbdcd727bd2" @@ -1964,7 +1991,7 @@ dependencies = [ "checksum rayon 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "655df67c314c30fa3055a365eae276eb88aa4f3413a352a1ab32c1320eda41ea" "checksum regex 0.1.68 (registry+https://github.com/rust-lang/crates.io-index)" = "b4329b8928a284580a1c63ec9d846b12f6d3472317243ff7077aff11f23f2b29" "checksum regex-syntax 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "841591b1e05609a643e3b4d0045fce04f701daba7151ddcd3ad47b080693d5a9" -"checksum ring 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d059a6a96d3be79042e3f70eb97945912839265f9d8ab45b921abaf266c70dbb" +"checksum ring 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0d2f6547bf9640f1d3cc4e771f82374ec8fd237c17eeb3ff5cd5ccbe22377a09" "checksum rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)" = "" "checksum rocksdb-sys 0.3.0 (git+https://github.com/ethcore/rust-rocksdb)" = "" "checksum rotor 0.6.3 (git+https://github.com/ethcore/rotor)" = "" @@ -1972,7 +1999,7 @@ dependencies = [ "checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" "checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b" "checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" -"checksum rustls 0.1.1 (git+https://github.com/ctz/rustls)" = "" +"checksum rustls 0.1.2 (git+https://github.com/ctz/rustls)" = "" "checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" "checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f" "checksum serde 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b1dfda9ebb31d29fa8b94d7eb3031a86a8dcec065f0fe268a30f98867bf45775" @@ -2017,7 +2044,7 @@ dependencies = [ "checksum vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0795a11576d29ae80525a3fda315bf7b534f8feb9d34101e5fe63fb95bb2fd24" "checksum vergen 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "56b639f935488eb40f06d17c3e3bcc3054f6f75d264e187b1107c8d1cba8d31c" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum webpki 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5dc10a815fabbb0c3145c1153240528f3a8703a47e26e8dbb4a5d4f6386200ad" +"checksum webpki 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "813503a5985585e0812d430cd1328ee322f47f66629c8ed4ecab939cf9e92f91" "checksum winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4dfaaa8fbdaa618fa6914b59b2769d690dd7521920a18d84b42d254678dd5fd4" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum ws 0.5.2 (git+https://github.com/ethcore/ws-rs.git?branch=mio-upstream-stable)" = "" diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index b1883e748..1019e5460 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -26,7 +26,7 @@ linked-hash-map = "0.3" ethcore-devtools = { path = "../devtools" } ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } -https-fetch = { path = "../util/https-fetch" } +fetch = { path = "../util/fetch" } parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } # List of apps parity-dapps-status = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" } diff --git a/dapps/src/handlers/client/mod.rs b/dapps/src/handlers/client/mod.rs deleted file mode 100644 index 3d8551e8a..000000000 --- a/dapps/src/handlers/client/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Hyper Client Handlers - -pub mod fetch_file; - -use std::env; -use std::sync::{mpsc, Arc}; -use std::sync::atomic::AtomicBool; -use std::path::PathBuf; - -use hyper; -use https_fetch as https; - -use random_filename; -use self::fetch_file::{Fetch, Error as HttpFetchError}; - -pub type FetchResult = Result; - -#[derive(Debug)] -pub enum FetchError { - InvalidUrl, - Http(HttpFetchError), - Https(https::FetchError), - Other(String), -} - -impl From for FetchError { - fn from(e: HttpFetchError) -> Self { - FetchError::Http(e) - } -} - -pub struct Client { - http_client: hyper::Client, - https_client: https::Client, -} - -impl Client { - pub fn new() -> Self { - Client { - http_client: hyper::Client::new().expect("Unable to initialize http client."), - https_client: https::Client::new().expect("Unable to initialize https client."), - } - } - - pub fn close(self) { - self.http_client.close(); - self.https_client.close(); - } - - pub fn request(&mut self, url: &str, abort: Arc, on_done: Box) -> Result, FetchError> { - let is_https = url.starts_with("https://"); - let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl)); - trace!(target: "dapps", "Fetching from: {:?}", url); - if is_https { - let url = try!(Self::convert_url(url)); - - let (tx, rx) = mpsc::channel(); - let temp_path = Self::temp_path(); - let res = self.https_client.fetch_to_file(url, temp_path.clone(), abort, move |result| { - let res = tx.send( - result.map(|_| temp_path).map_err(FetchError::Https) - ); - if let Err(_) = res { - warn!("Fetch finished, but no one was listening"); - } - on_done(); - }); - - match res { - Ok(_) => Ok(rx), - Err(e) => Err(FetchError::Other(format!("{:?}", e))), - } - } else { - let (tx, rx) = mpsc::channel(); - let res = self.http_client.request(url, Fetch::new(tx, abort, on_done)); - - match res { - Ok(_) => Ok(rx), - Err(e) => Err(FetchError::Other(format!("{:?}", e))), - } - } - } - - fn convert_url(url: hyper::Url) -> Result { - let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl))); - let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl)); - https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl) - } - - fn temp_path() -> PathBuf { - let mut dir = env::temp_dir(); - dir.push(random_filename()); - dir - } -} - - diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index c463d3710..639fc7497 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -22,13 +22,13 @@ use std::sync::{mpsc, Arc}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Instant, Duration}; use util::Mutex; +use fetch::{Client, Fetch, FetchResult}; use hyper::{server, Decoder, Encoder, Next, Method, Control}; use hyper::net::HttpStream; use hyper::status::StatusCode; use handlers::{ContentHandler, Redirection}; -use handlers::client::{Client, FetchResult}; use apps::redirection_address; use page::LocalPageEndpoint; @@ -159,7 +159,7 @@ impl ContentFetcherHandler { handler: H) -> (Self, Arc) { let fetch_control = Arc::new(FetchControl::default()); - let client = Client::new(); + let client = Client::default(); let handler = ContentFetcherHandler { fetch_control: fetch_control.clone(), control: Some(control), diff --git a/dapps/src/handlers/mod.rs b/dapps/src/handlers/mod.rs index 62b13eaa8..54644fe8d 100644 --- a/dapps/src/handlers/mod.rs +++ b/dapps/src/handlers/mod.rs @@ -21,7 +21,6 @@ mod echo; mod content; mod redirect; mod fetch; -pub mod client; pub use self::auth::AuthRequiredHandler; pub use self::echo::EchoHandler; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index edc0bebe5..cac42f893 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -58,10 +58,10 @@ extern crate jsonrpc_http_server; extern crate mime_guess; extern crate rustc_serialize; extern crate parity_dapps; -extern crate https_fetch; extern crate ethcore_rpc; extern crate ethcore_util as util; extern crate linked_hash_map; +extern crate fetch; #[cfg(test)] extern crate ethcore_devtools as devtools; diff --git a/ethash/src/compute.rs b/ethash/src/compute.rs index a99a0e3b5..6fcf17cf1 100644 --- a/ethash/src/compute.rs +++ b/ethash/src/compute.rs @@ -202,6 +202,9 @@ impl SeedHashCompute { } } +pub fn slow_get_seedhash(block_number: u64) -> H256 { + SeedHashCompute::resume_compute_seedhash([0u8; 32], 0, block_number / ETHASH_EPOCH_LENGTH) +} #[inline] fn fnv_hash(x: u32, y: u32) -> u32 { diff --git a/ethash/src/lib.rs b/ethash/src/lib.rs index 8fb2c43f6..130882318 100644 --- a/ethash/src/lib.rs +++ b/ethash/src/lib.rs @@ -26,7 +26,7 @@ mod compute; use std::mem; use compute::Light; -pub use compute::{ETHASH_EPOCH_LENGTH, H256, ProofOfWork, SeedHashCompute, quick_get_difficulty}; +pub use compute::{ETHASH_EPOCH_LENGTH, H256, ProofOfWork, SeedHashCompute, quick_get_difficulty, slow_get_seedhash}; use std::sync::Arc; use parking_lot::Mutex; diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 3ad9e69c4..2f1291d56 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -37,6 +37,7 @@ ethkey = { path = "../ethkey" } ethcore-ipc-nano = { path = "../ipc/nano" } rlp = { path = "../util/rlp" } rand = "0.3" +lru-cache = "0.0.7" [dependencies.hyper] git = "https://github.com/ethcore/hyper" diff --git a/ethcore/res/ethereum/expanse.json b/ethcore/res/ethereum/expanse.json new file mode 100644 index 000000000..9b005096b --- /dev/null +++ b/ethcore/res/ethereum/expanse.json @@ -0,0 +1,69 @@ +{ + "name": "Expanse", + "forkName": "expanse", + "engine": { + "Ethash": { + "params": { + "gasLimitBoundDivisor": "0x0400", + "minimumDifficulty": "0x020000", + "difficultyBoundDivisor": "0x0800", + "difficultyIncrementDivisor": "60", + "durationLimit": "0x3C", + "blockReward": "0x6f05b59d3b200000", + "registrar" : "0x6c221ca53705f3497ec90ca7b84c59ae7382fc21", + "frontierCompatibilityModeLimit": "0x30d40", + "difficultyHardforkTransition": "0x59d9", + "difficultyHardforkBoundDivisor": "0x0200", + "bombDefuseTransition": "0x30d40" + } + } + }, + "params": { + "accountStartNonce": "0x00", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID": "0x1", + "subprotocolName": "exp" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x214652414e4b4f21", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty": "0x40000000", + "author": "0x93decab0cd745598860f782ac1e8f046cb99e898", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x4672616e6b6f497346726565646f6d", + "gasLimit": "0x1388" + }, + "nodes": [ + "enode://7f335a047654f3e70d6f91312a7cf89c39704011f1a584e2698250db3d63817e74b88e26b7854111e16b2c9d0c7173c05419aeee2d0321850227b126d8b1be3f@46.101.156.249:42786", + "enode://df872f81e25f72356152b44cab662caf1f2e57c3a156ecd20e9ac9246272af68a2031b4239a0bc831f2c6ab34733a041464d46b3ea36dce88d6c11714446e06b@178.62.208.109:42786", + "enode://96d3919b903e7f5ad59ac2f73c43be9172d9d27e2771355db03fd194732b795829a31fe2ea6de109d0804786c39a807e155f065b4b94c6fce167becd0ac02383@45.55.22.34:42786", + "enode://5f6c625bf287e3c08aad568de42d868781e961cbda805c8397cfb7be97e229419bef9a5a25a75f97632787106bba8a7caf9060fab3887ad2cfbeb182ab0f433f@46.101.182.53:42786", + "enode://d33a8d4c2c38a08971ed975b750f21d54c927c0bf7415931e214465a8d01651ecffe4401e1db913f398383381413c78105656d665d83f385244ab302d6138414@128.199.183.48:42786", + "enode://df872f81e25f72356152b44cab662caf1f2e57c3a156ecd20e9ac9246272af68a2031b4239a0bc831f2c6ab34733a041464d46b3ea36dce88d6c11714446e06b@178.62.208.109:42786", + "enode://f6f0d6b9b7d02ec9e8e4a16e38675f3621ea5e69860c739a65c1597ca28aefb3cec7a6d84e471ac927d42a1b64c1cbdefad75e7ce8872d57548ddcece20afdd1@159.203.64.95:42786" + ], + "accounts": { + "0000000000000000000000000000000000000001": { "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "bb94f0ceb32257275b2a7a9c094c13e469b4563e": { + "balance": "10000000000000000000000000" + }, + "15656715068ab0dbdf0ab00748a8a19e40f28192": { + "balance": "1000000000000000000000000" + }, + "c075fa11f85bda3aaba67106226aaf086ac16f4e": { + "balance": "100000000000000000000000" + }, + "93decab0cd745598860f782ac1e8f046cb99e898": { + "balance": "10000000000000000000000" + } + } +} diff --git a/ethcore/res/ethereum/frontier.json b/ethcore/res/ethereum/frontier.json index 2f91809fc..903e87cc7 100644 --- a/ethcore/res/ethereum/frontier.json +++ b/ethcore/res/ethereum/frontier.json @@ -157,6 +157,9 @@ "stateRoot": "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544" }, "nodes": [ + "enode://cd6611461840543d5b9c56fbf088736154c699c43973b3a1a32390cf27106f87e58a818a606ccb05f3866de95a4fe860786fea71bf891ea95f234480d3022aa3@136.243.154.245:30303", + "enode://bcc7240543fe2cf86f5e9093d05753dd83343f8fda7bf0e833f65985c73afccf8f981301e13ef49c4804491eab043647374df1c4adf85766af88a624ecc3330e@136.243.154.244:30303", + "enode://ed4227681ca8c70beb2277b9e870353a9693f12e7c548c35df6bca6a956934d6f659999c2decb31f75ce217822eefca149ace914f1cbe461ed5a2ebaf9501455@88.212.206.70:30303", "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", "enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303", "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index b35b4dc1a..dc60f9bbe 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -19,6 +19,7 @@ use common::*; use engines::Engine; use state::*; +use state_db::StateDB; use verification::PreverifiedBlock; use trace::FlatTrace; use factory::Factories; @@ -69,7 +70,7 @@ impl Decodable for Block { } } -/// Internal type for a block's common elements. +/// An internal type for a block's common elements. #[derive(Clone)] pub struct ExecutedBlock { base: Block, @@ -179,7 +180,7 @@ pub trait IsBlock { /// Trait for a object that has a state database. pub trait Drain { /// Drop this object and return the underlieing database. - fn drain(self) -> Box; + fn drain(self) -> StateDB; } impl IsBlock for ExecutedBlock { @@ -205,6 +206,7 @@ pub struct ClosedBlock { block: ExecutedBlock, uncle_bytes: Bytes, last_hashes: Arc, + unclosed_state: State, } /// Just like `ClosedBlock` except that we can't reopen it and it's faster. @@ -231,7 +233,7 @@ impl<'x> OpenBlock<'x> { engine: &'x Engine, factories: Factories, tracing: bool, - db: Box, + db: StateDB, parent: &Header, last_hashes: Arc, author: Address, @@ -346,8 +348,7 @@ impl<'x> OpenBlock<'x> { pub fn close(self) -> ClosedBlock { let mut s = self; - // take a snapshot so the engine's changes can be rolled back. - s.block.state.snapshot(); + let unclosed_state = s.block.state.clone(); s.engine.on_close_block(&mut s.block); s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()))); @@ -362,6 +363,7 @@ impl<'x> OpenBlock<'x> { block: s.block, uncle_bytes: uncle_bytes, last_hashes: s.last_hashes, + unclosed_state: unclosed_state, } } @@ -369,9 +371,6 @@ impl<'x> OpenBlock<'x> { pub fn close_and_lock(self) -> LockedBlock { let mut s = self; - // take a snapshot so the engine's changes can be rolled back. - s.block.state.snapshot(); - s.engine.on_close_block(&mut s.block); if s.block.base.header.transactions_root().is_zero() || s.block.base.header.transactions_root() == &SHA3_NULL_RLP { s.block.base.header.set_transactions_root(ordered_trie_root(s.block.base.transactions.iter().map(|e| e.rlp_bytes().to_vec()))); @@ -388,11 +387,10 @@ impl<'x> OpenBlock<'x> { s.block.base.header.set_log_bloom(s.block.receipts.iter().fold(LogBloom::zero(), |mut b, r| {b = &b | &r.log_bloom; b})); //TODO: use |= operator s.block.base.header.set_gas_used(s.block.receipts.last().map_or(U256::zero(), |r| r.gas_used)); - ClosedBlock { + LockedBlock { block: s.block, uncle_bytes: uncle_bytes, - last_hashes: s.last_hashes, - }.lock() + } } } @@ -413,17 +411,7 @@ impl ClosedBlock { pub fn hash(&self) -> H256 { self.header().rlp_sha3(Seal::Without) } /// Turn this into a `LockedBlock`, unable to be reopened again. - pub fn lock(mut self) -> LockedBlock { - // finalize the changes made by the engine. - self.block.state.clear_snapshot(); - if let Err(e) = self.block.state.commit() { - warn!("Error committing closed block's state: {:?}", e); - } - - // set the state root here, after commit recalculates with the block - // rewards. - self.block.base.header.set_state_root(self.block.state.root().clone()); - + pub fn lock(self) -> LockedBlock { LockedBlock { block: self.block, uncle_bytes: self.uncle_bytes, @@ -431,12 +419,12 @@ impl ClosedBlock { } /// Given an engine reference, reopen the `ClosedBlock` into an `OpenBlock`. - pub fn reopen(mut self, engine: &Engine) -> OpenBlock { + pub fn reopen(self, engine: &Engine) -> OpenBlock { // revert rewards (i.e. set state back at last transaction's state). - self.block.state.revert_snapshot(); - + let mut block = self.block; + block.state = self.unclosed_state; OpenBlock { - block: self.block, + block: block, engine: engine, last_hashes: self.last_hashes, } @@ -462,11 +450,11 @@ impl LockedBlock { /// Provide a valid seal in order to turn this into a `SealedBlock`. /// This does check the validity of `seal` with the engine. /// Returns the `ClosedBlock` back again if the seal is no good. - pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { + pub fn try_seal(self, engine: &Engine, seal: Vec) -> Result { let mut s = self; s.block.base.header.set_seal(seal); match engine.verify_block_seal(&s.block.base.header) { - Err(_) => Err(s), + Err(e) => Err((e, s)), _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } } @@ -474,7 +462,9 @@ impl LockedBlock { impl Drain for LockedBlock { /// Drop this object and return the underlieing database. - fn drain(self) -> Box { self.block.state.drop().1 } + fn drain(self) -> StateDB { + self.block.state.drop().1 + } } impl SealedBlock { @@ -490,7 +480,9 @@ impl SealedBlock { impl Drain for SealedBlock { /// Drop this object and return the underlieing database. - fn drain(self) -> Box { self.block.state.drop().1 } + fn drain(self) -> StateDB { + self.block.state.drop().1 + } } impl IsBlock for SealedBlock { @@ -505,7 +497,7 @@ pub fn enact( uncles: &[Header], engine: &Engine, tracing: bool, - db: Box, + db: StateDB, parent: &Header, last_hashes: Arc, factories: Factories, @@ -537,7 +529,7 @@ pub fn enact_bytes( block_bytes: &[u8], engine: &Engine, tracing: bool, - db: Box, + db: StateDB, parent: &Header, last_hashes: Arc, factories: Factories, @@ -553,7 +545,7 @@ pub fn enact_verified( block: &PreverifiedBlock, engine: &Engine, tracing: bool, - db: Box, + db: StateDB, parent: &Header, last_hashes: Arc, factories: Factories, @@ -568,7 +560,7 @@ pub fn enact_and_seal( block_bytes: &[u8], engine: &Engine, tracing: bool, - db: Box, + db: StateDB, parent: &Header, last_hashes: Arc, factories: Factories, @@ -588,9 +580,9 @@ mod tests { use spec::*; let spec = Spec::new_test(); let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let b = OpenBlock::new(&*spec.engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); @@ -604,25 +596,25 @@ mod tests { let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap() .close_and_lock().seal(engine, vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap(); assert_eq!(e.rlp_bytes(), orig_bytes); let db = e.drain(); - assert_eq!(orig_db.keys(), db.keys()); - assert!(orig_db.keys().iter().filter(|k| orig_db.get(k.0) != db.get(k.0)).next() == None); + assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys()); + assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None); } #[test] @@ -632,9 +624,9 @@ mod tests { let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let mut open_block = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let mut uncle1_header = Header::new(); @@ -648,9 +640,9 @@ mod tests { let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let e = enact_and_seal(&orig_bytes, engine, false, db, &genesis_header, last_hashes, Default::default()).unwrap(); let bytes = e.rlp_bytes(); @@ -659,7 +651,7 @@ mod tests { assert_eq!(uncles[1].extra_data(), b"uncle2"); let db = e.drain(); - assert_eq!(orig_db.keys(), db.keys()); - assert!(orig_db.keys().iter().filter(|k| orig_db.get(k.0) != db.get(k.0)).next() == None); + assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys()); + assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None); } } diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 392581fd1..8daf672b9 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -181,7 +181,7 @@ pub struct BlockChain { pending_best_block: RwLock>, pending_block_hashes: RwLock>, - pending_transaction_addresses: RwLock>, + pending_transaction_addresses: RwLock>>, } impl BlockProvider for BlockChain { @@ -331,11 +331,12 @@ impl BlockProvider for BlockChain { .filter_map(|number| self.block_hash(number).map(|hash| (number, hash))) .filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts))) .filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, BodyView::new(b).transaction_hashes()))) - .flat_map(|(number, hash, mut receipts, hashes)| { + .flat_map(|(number, hash, mut receipts, mut hashes)| { assert_eq!(receipts.len(), hashes.len()); log_index = receipts.iter().fold(0, |sum, receipt| sum + receipt.logs.len()); let receipts_len = receipts.len(); + hashes.reverse(); receipts.reverse(); receipts.into_iter() .map(|receipt| receipt.logs) @@ -680,8 +681,8 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: self.prepare_block_details_update(bytes, &info), block_receipts: self.prepare_block_receipts_update(receipts, &info), - transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), blocks_blooms: self.prepare_block_blooms_update(bytes, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info, block: bytes }, is_best); @@ -714,8 +715,8 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: update, block_receipts: self.prepare_block_receipts_update(receipts, &info), - transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), blocks_blooms: self.prepare_block_blooms_update(bytes, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info, block: bytes, }, is_best); @@ -783,8 +784,8 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: self.prepare_block_details_update(bytes, &info), block_receipts: self.prepare_block_receipts_update(receipts, &info), - transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), blocks_blooms: self.prepare_block_blooms_update(bytes, &info), + transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info.clone(), block: bytes, }, true); @@ -877,7 +878,7 @@ impl BlockChain { let mut write_txs = self.pending_transaction_addresses.write(); batch.extend_with_cache(db::COL_EXTRA, &mut *write_hashes, update.block_hashes, CacheUpdatePolicy::Overwrite); - batch.extend_with_cache(db::COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Overwrite); + batch.extend_with_option_cache(db::COL_EXTRA, &mut *write_txs, update.transactions_addresses, CacheUpdatePolicy::Overwrite); } } @@ -895,18 +896,25 @@ impl BlockChain { *best_block = block; } + let pending_txs = mem::replace(&mut *pending_write_txs, HashMap::new()); + let (retracted_txs, enacted_txs) = pending_txs.into_iter().partition::, _>(|&(_, ref value)| value.is_none()); + let pending_hashes_keys: Vec<_> = pending_write_hashes.keys().cloned().collect(); - let pending_txs_keys: Vec<_> = pending_write_txs.keys().cloned().collect(); + let enacted_txs_keys: Vec<_> = enacted_txs.keys().cloned().collect(); write_hashes.extend(mem::replace(&mut *pending_write_hashes, HashMap::new())); - write_txs.extend(mem::replace(&mut *pending_write_txs, HashMap::new())); + write_txs.extend(enacted_txs.into_iter().map(|(k, v)| (k, v.expect("Transactions were partitioned; qed")))); + + for hash in retracted_txs.keys() { + write_txs.remove(hash); + } let mut cache_man = self.cache_man.lock(); for n in pending_hashes_keys { cache_man.note_used(CacheID::BlockHashes(n)); } - for hash in pending_txs_keys { + for hash in enacted_txs_keys { cache_man.note_used(CacheID::TransactionAddresses(hash)); } } @@ -1008,7 +1016,7 @@ impl BlockChain { } /// This function returns modified transaction addresses. - fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { + fn prepare_transaction_addresses_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap> { let block = BlockView::new(block_bytes); let transaction_hashes = block.transaction_hashes(); @@ -1017,10 +1025,10 @@ impl BlockChain { transaction_hashes.into_iter() .enumerate() .map(|(i ,tx_hash)| { - (tx_hash, TransactionAddress { + (tx_hash, Some(TransactionAddress { block_hash: info.hash.clone(), index: i - }) + })) }) .collect() }, @@ -1031,23 +1039,30 @@ impl BlockChain { let hashes = BodyView::new(&bytes).transaction_hashes(); hashes.into_iter() .enumerate() - .map(|(i, tx_hash)| (tx_hash, TransactionAddress { + .map(|(i, tx_hash)| (tx_hash, Some(TransactionAddress { block_hash: hash.clone(), index: i, - })) - .collect::>() + }))) + .collect::>>() }); let current_addresses = transaction_hashes.into_iter() .enumerate() .map(|(i ,tx_hash)| { - (tx_hash, TransactionAddress { + (tx_hash, Some(TransactionAddress { block_hash: info.hash.clone(), index: i - }) + })) }); - addresses.chain(current_addresses).collect() + let retracted = data.retracted.iter().flat_map(|hash| { + let bytes = self.block_body(hash).expect("Retracted block must be in database."); + let hashes = BodyView::new(&bytes).transaction_hashes(); + hashes.into_iter().map(|hash| (hash, None)).collect::>>() + }); + + // The order here is important! Don't remove transaction if it was part of enacted blocks as well. + retracted.chain(addresses).chain(current_addresses).collect() }, BlockLocation::Branch => HashMap::new(), } @@ -1345,6 +1360,70 @@ mod tests { // TODO: insert block that already includes one of them as an uncle to check it's not allowed. } + #[test] + fn test_fork_transaction_addresses() { + let mut canon_chain = ChainGenerator::default(); + let mut finalizer = BlockFinalizer::default(); + let genesis = canon_chain.generate(&mut finalizer).unwrap(); + let mut fork_chain = canon_chain.fork(1); + let mut fork_finalizer = finalizer.fork(); + + let t1 = Transaction { + nonce: 0.into(), + gas_price: 0.into(), + gas: 100_000.into(), + action: Action::Create, + value: 100.into(), + data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(), + }.sign(&"".sha3()); + + + let b1a = canon_chain + .with_transaction(t1.clone()) + .generate(&mut finalizer).unwrap(); + + // Empty block + let b1b = fork_chain + .generate(&mut fork_finalizer).unwrap(); + + let b2 = fork_chain + .generate(&mut fork_finalizer).unwrap(); + + let b1a_hash = BlockView::new(&b1a).header_view().sha3(); + let b2_hash = BlockView::new(&b2).header_view().sha3(); + + let t1_hash = t1.hash(); + + let temp = RandomTempPath::new(); + let db = new_db(temp.as_str()); + let bc = BlockChain::new(Config::default(), &genesis, db.clone()); + + let mut batch = db.transaction(); + let _ = bc.insert_block(&mut batch, &b1a, vec![]); + bc.commit(); + let _ = bc.insert_block(&mut batch, &b1b, vec![]); + bc.commit(); + db.write(batch).unwrap(); + + assert_eq!(bc.best_block_hash(), b1a_hash); + assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress { + block_hash: b1a_hash.clone(), + index: 0, + })); + + // now let's make forked chain the canon chain + let mut batch = db.transaction(); + let _ = bc.insert_block(&mut batch, &b2, vec![]); + bc.commit(); + db.write(batch).unwrap(); + + // Transaction should be retracted + assert_eq!(bc.best_block_hash(), b2_hash); + assert_eq!(bc.transaction_address(&t1_hash), None); + } + + + #[test] fn test_overwriting_transaction_addresses() { let mut canon_chain = ChainGenerator::default(); @@ -1415,14 +1494,14 @@ mod tests { db.write(batch).unwrap(); assert_eq!(bc.best_block_hash(), b1a_hash); - assert_eq!(bc.transaction_address(&t1_hash).unwrap(), TransactionAddress { + assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress { block_hash: b1a_hash.clone(), index: 0, - }); - assert_eq!(bc.transaction_address(&t2_hash).unwrap(), TransactionAddress { + })); + assert_eq!(bc.transaction_address(&t2_hash), Some(TransactionAddress { block_hash: b1a_hash.clone(), index: 1, - }); + })); // now let's make forked chain the canon chain let mut batch = db.transaction(); @@ -1431,18 +1510,18 @@ mod tests { db.write(batch).unwrap(); assert_eq!(bc.best_block_hash(), b2_hash); - assert_eq!(bc.transaction_address(&t1_hash).unwrap(), TransactionAddress { + assert_eq!(bc.transaction_address(&t1_hash), Some(TransactionAddress { block_hash: b1b_hash.clone(), index: 1, - }); - assert_eq!(bc.transaction_address(&t2_hash).unwrap(), TransactionAddress { + })); + assert_eq!(bc.transaction_address(&t2_hash), Some(TransactionAddress { block_hash: b1b_hash.clone(), index: 0, - }); - assert_eq!(bc.transaction_address(&t3_hash).unwrap(), TransactionAddress { + })); + assert_eq!(bc.transaction_address(&t3_hash), Some(TransactionAddress { block_hash: b2_hash.clone(), index: 0, - }); + })); } #[test] @@ -1682,7 +1761,7 @@ mod tests { gas_price: 0.into(), gas: 100_000.into(), action: Action::Create, - value: 100.into(), + value: 101.into(), data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(), }.sign(&"".sha3()); let t2 = Transaction { @@ -1690,7 +1769,7 @@ mod tests { gas_price: 0.into(), gas: 100_000.into(), action: Action::Create, - value: 100.into(), + value: 102.into(), data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(), }.sign(&"".sha3()); let t3 = Transaction { @@ -1698,7 +1777,7 @@ mod tests { gas_price: 0.into(), gas: 100_000.into(), action: Action::Create, - value: 100.into(), + value: 103.into(), data: "601080600c6000396000f3006000355415600957005b60203560003555".from_hex().unwrap(), }.sign(&"".sha3()); let tx_hash1 = t1.hash(); diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs index 24d0644e8..0d7d1dbed 100644 --- a/ethcore/src/blockchain/update.rs +++ b/ethcore/src/blockchain/update.rs @@ -17,8 +17,8 @@ pub struct ExtrasUpdate<'a> { pub block_details: HashMap, /// Modified block receipts. pub block_receipts: HashMap, - /// Modified transaction addresses. - pub transactions_addresses: HashMap, /// Modified blocks blooms. pub blocks_blooms: HashMap, + /// Modified transaction addresses (None signifies removed transactions). + pub transactions_addresses: HashMap>, } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 0e662aa77..541bd7872 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -32,7 +32,7 @@ use util::kvdb::*; // other use ethkey::recover; use io::*; -use views::{BlockView, HeaderView, BodyView}; +use views::{HeaderView, BodyView}; use error::{ImportError, ExecutionError, CallError, BlockError, ImportResult, Error as EthcoreError}; use header::BlockNumber; use state::State; @@ -48,7 +48,7 @@ use transaction::{LocalizedTransaction, SignedTransaction, Action}; use blockchain::extras::TransactionAddress; use types::filter::Filter; use log_entry::LocalizedLogEntry; -use block_queue::{BlockQueue, BlockQueueInfo}; +use verification::queue::{BlockQueue, QueueInfo as BlockQueueInfo}; use blockchain::{BlockChain, BlockProvider, TreeRoute, ImportRoute}; use client::{ BlockID, TransactionID, UncleID, TraceId, ClientConfig, BlockChainClient, @@ -67,7 +67,7 @@ use miner::{Miner, MinerService}; use snapshot::{self, io as snapshot_io}; use factory::Factories; use rlp::{View, UntrustedRlp}; - +use state_db::StateDB; // re-export pub use types::blockchain_info::BlockChainInfo; @@ -127,9 +127,9 @@ pub struct Client { tracedb: RwLock>, engine: Arc, config: ClientConfig, - db: RwLock>, pruning: journaldb::Algorithm, - state_db: RwLock>, + db: RwLock>, + state_db: Mutex, block_queue: BlockQueue, report: RwLock, import_lock: Mutex<()>, @@ -173,14 +173,15 @@ impl Client { let chain = Arc::new(BlockChain::new(config.blockchain.clone(), &gb, db.clone())); let tracedb = RwLock::new(TraceDB::new(config.tracing.clone(), db.clone(), chain.clone())); - let mut state_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE); - if state_db.is_empty() && try!(spec.ensure_db_good(state_db.as_hashdb_mut())) { + let journal_db = journaldb::new(db.clone(), config.pruning, ::db::COL_STATE); + let mut state_db = StateDB::new(journal_db); + if state_db.journal_db().is_empty() && try!(spec.ensure_db_good(&mut state_db)) { let mut batch = DBTransaction::new(&db); try!(state_db.commit(&mut batch, 0, &spec.genesis_header().hash(), None)); try!(db.write(batch).map_err(ClientError::Database)); } - if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.contains(h.state_root())) { + if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(h.state_root())) { warn!("State root not found for block #{} ({})", chain.best_block_number(), chain.best_block_hash().hex()); } @@ -209,7 +210,7 @@ impl Client { verifier: verification::new(config.verifier_type.clone()), config: config, db: RwLock::new(db), - state_db: RwLock::new(state_db), + state_db: Mutex::new(state_db), block_queue: block_queue, report: RwLock::new(Default::default()), import_lock: Mutex::new(()), @@ -300,7 +301,8 @@ impl Client { // Enact Verified Block let parent = chain_has_parent.unwrap(); let last_hashes = self.build_last_hashes(header.parent_hash().clone()); - let db = self.state_db.read().boxed_clone(); + let is_canon = header.parent_hash() == &chain.best_block_hash(); + let db = if is_canon { self.state_db.lock().boxed_clone_canon() } else { self.state_db.lock().boxed_clone() }; let enact_result = enact_verified(block, engine, self.tracedb.read().tracing_enabled(), db, &parent, last_hashes, self.factories.clone()); if let Err(e) = enact_result { @@ -443,7 +445,8 @@ impl Client { // CHECK! I *think* this is fine, even if the state_root is equal to another // already-imported block of the same number. // TODO: Prove it with a test. - block.drain().commit(&mut batch, number, hash, ancient).expect("DB commit failed."); + let mut state = block.drain(); + state.commit(&mut batch, number, hash, ancient).expect("DB commit failed."); let route = chain.insert_block(&mut batch, block_data, receipts); self.tracedb.read().import(&mut batch, TraceImportRequest { @@ -456,7 +459,6 @@ impl Client { // Final commit to the DB self.db.read().write_buffered(batch); chain.commit(); - self.update_last_hashes(&parent, hash); route } @@ -498,7 +500,7 @@ impl Client { }; self.block_header(id).and_then(|header| { - let db = self.state_db.read().boxed_clone(); + let db = self.state_db.lock().boxed_clone(); // early exit for pruned blocks if db.is_pruned() && self.chain.read().best_block_number() >= block_number + HISTORY { @@ -529,7 +531,7 @@ impl Client { /// Get a copy of the best block's state. pub fn state(&self) -> State { State::from_existing( - self.state_db.read().boxed_clone(), + self.state_db.lock().boxed_clone(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce(), self.factories.clone()) @@ -544,7 +546,7 @@ impl Client { /// Get the report. pub fn report(&self) -> ClientReport { let mut report = self.report.read().clone(); - report.state_db_mem = self.state_db.read().mem_used(); + report.state_db_mem = self.state_db.lock().mem_used(); report } @@ -600,7 +602,7 @@ impl Client { /// Take a snapshot at the given block. /// If the ID given is "latest", this will default to 1000 blocks behind. pub fn take_snapshot(&self, writer: W, at: BlockID, p: &snapshot::Progress) -> Result<(), EthcoreError> { - let db = self.state_db.read().boxed_clone(); + let db = self.state_db.lock().journal_db().boxed_clone(); let best_block_number = self.chain_info().best_block_number; let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at))); @@ -679,14 +681,14 @@ impl snapshot::DatabaseRestore for Client { trace!(target: "snapshot", "Replacing client database with {:?}", new_db); let _import_lock = self.import_lock.lock(); - let mut state_db = self.state_db.write(); + let mut state_db = self.state_db.lock(); let mut chain = self.chain.write(); let mut tracedb = self.tracedb.write(); self.miner.clear(); let db = self.db.write(); try!(db.restore(new_db)); - *state_db = journaldb::new(db.clone(), self.pruning, ::db::COL_STATE); + *state_db = StateDB::new(journaldb::new(db.clone(), self.pruning, ::db::COL_STATE)); *chain = Arc::new(BlockChain::new(self.config.blockchain.clone(), &[], db.clone())); *tracedb = TraceDB::new(self.config.tracing.clone(), db.clone(), chain.clone()); Ok(()) @@ -806,7 +808,7 @@ impl BlockChainClient for Client { let chain = self.chain.read(); match Self::block_hash(&chain, id) { Some(ref hash) if chain.is_known(hash) => BlockStatus::InChain, - Some(hash) => self.block_queue.block_status(&hash), + Some(hash) => self.block_queue.status(&hash).into(), None => BlockStatus::Unknown } } @@ -853,9 +855,12 @@ impl BlockChainClient for Client { fn transaction_receipt(&self, id: TransactionID) -> Option { let chain = self.chain.read(); - self.transaction_address(id).and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| { + self.transaction_address(id) + .and_then(|address| chain.block_number(&address.block_hash).and_then(|block_number| { let t = chain.block_body(&address.block_hash) - .and_then(|block| BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index)); + .and_then(|block| { + BodyView::new(&block).localized_transaction_at(&address.block_hash, block_number, address.index) + }); match (t, chain.transaction_receipt(&address)) { (Some(tx), Some(receipt)) => { @@ -910,7 +915,7 @@ impl BlockChainClient for Client { } fn state_data(&self, hash: &H256) -> Option { - self.state_db.read().state(hash) + self.state_db.lock().journal_db().state(hash) } fn block_receipts(&self, hash: &H256) -> Option { @@ -918,16 +923,21 @@ impl BlockChainClient for Client { } fn import_block(&self, bytes: Bytes) -> Result { + use verification::queue::kind::HasHash; + use verification::queue::kind::blocks::Unverified; + + // create unverified block here so the `sha3` calculation can be cached. + let unverified = Unverified::new(bytes); + { - let header = BlockView::new(&bytes).header_view(); - if self.chain.read().is_known(&header.sha3()) { + if self.chain.read().is_known(&unverified.hash()) { return Err(BlockImportError::Import(ImportError::AlreadyInChain)); } - if self.block_status(BlockID::Hash(header.parent_hash())) == BlockStatus::Unknown { - return Err(BlockImportError::Block(BlockError::UnknownParent(header.parent_hash()))); + if self.block_status(BlockID::Hash(unverified.parent_hash())) == BlockStatus::Unknown { + return Err(BlockImportError::Block(BlockError::UnknownParent(unverified.parent_hash()))); } } - Ok(try!(self.block_queue.import_block(bytes))) + Ok(try!(self.block_queue.import(unverified))) } fn queue_info(&self) -> BlockQueueInfo { @@ -1062,7 +1072,7 @@ impl MiningBlockChainClient for Client { engine, self.factories.clone(), false, // TODO: this will need to be parameterised once we want to do immediate mining insertion. - self.state_db.read().boxed_clone(), + self.state_db.lock().boxed_clone(), &chain.block_header(&h).expect("h is best block hash: so its header must exist: qed"), self.build_last_hashes(h.clone()), author, diff --git a/ethcore/src/client/config.rs b/ethcore/src/client/config.rs index 0146293df..399132108 100644 --- a/ethcore/src/client/config.rs +++ b/ethcore/src/client/config.rs @@ -16,11 +16,11 @@ use std::str::FromStr; pub use std::time::Duration; -pub use block_queue::BlockQueueConfig; pub use blockchain::Config as BlockChainConfig; pub use trace::Config as TraceConfig; pub use evm::VMType; -pub use verification::VerifierType; + +use verification::{VerifierType, QueueConfig}; use util::{journaldb, CompactionProfile}; use util::trie::TrieSpec; @@ -84,7 +84,7 @@ impl Default for Mode { #[derive(Debug, PartialEq, Default)] pub struct ClientConfig { /// Block queue configuration. - pub queue: BlockQueueConfig, + pub queue: QueueConfig, /// Blockchain configuration. pub blockchain: BlockChainConfig, /// Trace configuration. diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index a5ff89c47..3bbf9011b 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -23,7 +23,7 @@ mod trace; mod client; pub use self::client::*; -pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockQueueConfig, BlockChainConfig, VMType}; +pub use self::config::{Mode, ClientConfig, DatabaseCompactionProfile, BlockChainConfig, VMType}; pub use self::error::Error; pub use types::ids::*; pub use self::test_client::{TestBlockChainClient, EachBlockWith}; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 2e5b9365d..cd6d43a8a 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -37,11 +37,12 @@ use evm::{Factory as EvmFactory, VMType}; use miner::{Miner, MinerService, TransactionImportResult}; use spec::Spec; -use block_queue::BlockQueueInfo; +use verification::queue::QueueInfo; use block::{OpenBlock, SealedBlock}; use executive::Executed; use error::CallError; use trace::LocalizedTrace; +use state_db::StateDB; /// Test client. pub struct TestBlockChainClient { @@ -283,13 +284,14 @@ impl TestBlockChainClient { } } -pub fn get_temp_journal_db() -> GuardedTempResult> { +pub fn get_temp_state_db() -> GuardedTempResult { let temp = RandomTempPath::new(); let db = Database::open_default(temp.as_str()).unwrap(); let journal_db = journaldb::new(Arc::new(db), journaldb::Algorithm::EarlyMerge, None); + let state_db = StateDB::new(journal_db); GuardedTempResult { _temp: temp, - result: Some(journal_db) + result: Some(state_db) } } @@ -297,9 +299,9 @@ impl MiningBlockChainClient for TestBlockChainClient { fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock { let engine = &*self.spec.engine; let genesis_header = self.spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - self.spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + self.spec.ensure_db_good(&mut db).unwrap(); let last_hashes = vec![genesis_header.hash()]; let mut open_block = OpenBlock::new( @@ -383,11 +385,11 @@ impl BlockChainClient for TestBlockChainClient { } fn transaction(&self, _id: TransactionID) -> Option { - unimplemented!(); + None // Simple default. } fn uncle(&self, _id: UncleID) -> Option { - unimplemented!(); + None // Simple default. } fn transaction_receipt(&self, id: TransactionID) -> Option { @@ -544,8 +546,8 @@ impl BlockChainClient for TestBlockChainClient { Ok(h) } - fn queue_info(&self) -> BlockQueueInfo { - BlockQueueInfo { + fn queue_info(&self) -> QueueInfo { + QueueInfo { verified_queue_size: self.queue_size.load(AtomicOrder::Relaxed), unverified_queue_size: 0, verifying_queue_size: 0, diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index fa2b56afe..fb63d8c17 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; use util::{U256, Address, H256, H2048, Bytes, Itertools}; use blockchain::TreeRoute; -use block_queue::BlockQueueInfo; +use verification::queue::QueueInfo as BlockQueueInfo; use block::{OpenBlock, SealedBlock}; use header::{BlockNumber}; use transaction::{LocalizedTransaction, SignedTransaction}; diff --git a/ethcore/src/db.rs b/ethcore/src/db.rs index 61cd41bd6..c8c24cc5f 100644 --- a/ethcore/src/db.rs +++ b/ethcore/src/db.rs @@ -86,6 +86,9 @@ pub trait Writable { /// Writes the value into the database. fn write(&mut self, col: Option, key: &Key, value: &T) where T: rlp::Encodable, R: Deref; + /// Deletes key from the databse. + fn delete(&mut self, col: Option, key: &Key) where T: rlp::Encodable, R: Deref; + /// Writes the value into the database and updates the cache. fn write_with_cache(&mut self, col: Option, cache: &mut Cache, key: K, value: T, policy: CacheUpdatePolicy) where K: Key + Hash + Eq, @@ -122,6 +125,34 @@ pub trait Writable { }, } } + + /// Writes and removes the values into the database and updates the cache. + fn extend_with_option_cache(&mut self, col: Option, cache: &mut Cache>, values: HashMap>, policy: CacheUpdatePolicy) where + K: Key + Hash + Eq, + T: rlp::Encodable, + R: Deref { + match policy { + CacheUpdatePolicy::Overwrite => { + for (key, value) in values.into_iter() { + match value { + Some(ref v) => self.write(col, &key, v), + None => self.delete(col, &key), + } + cache.insert(key, value); + } + }, + CacheUpdatePolicy::Remove => { + for (key, value) in values.into_iter() { + match value { + Some(v) => self.write(col, &key, &v), + None => self.delete(col, &key), + } + cache.remove(&key); + } + }, + } + } + } /// Should be used to read values from database. @@ -173,6 +204,10 @@ impl Writable for DBTransaction { fn write(&mut self, col: Option, key: &Key, value: &T) where T: rlp::Encodable, R: Deref { self.put(col, &key.key(), &rlp::encode(value)); } + + fn delete(&mut self, col: Option, key: &Key) where T: rlp::Encodable, R: Deref { + self.delete(col, &key.key()); + } } impl Readable for Database { diff --git a/ethcore/src/engines/basic_authority.rs b/ethcore/src/engines/basic_authority.rs index 8d852e99e..6e3c2f1dd 100644 --- a/ethcore/src/engines/basic_authority.rs +++ b/ethcore/src/engines/basic_authority.rs @@ -252,9 +252,9 @@ mod tests { let spec = new_test_authority(); let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); diff --git a/ethcore/src/engines/instant_seal.rs b/ethcore/src/engines/instant_seal.rs index 26d2ed5bf..174a80ea8 100644 --- a/ethcore/src/engines/instant_seal.rs +++ b/ethcore/src/engines/instant_seal.rs @@ -81,9 +81,9 @@ mod tests { let spec = Spec::new_test_instant(); let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close_and_lock(); diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 734acb758..982698a50 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethash::{quick_get_difficulty, EthashManager, H256 as EH256}; +use ethash::{quick_get_difficulty, slow_get_seedhash, EthashManager, H256 as EH256}; use common::*; use block::*; use spec::CommonParams; @@ -32,6 +32,8 @@ pub struct EthashParams { pub minimum_difficulty: U256, /// Difficulty bound divisor. pub difficulty_bound_divisor: U256, + /// Difficulty increment divisor. + pub difficulty_increment_divisor: u64, /// Block duration. pub duration_limit: u64, /// Block reward. @@ -46,6 +48,12 @@ pub struct EthashParams { pub dao_hardfork_beneficiary: Address, /// DAO hard-fork DAO accounts list (L) pub dao_hardfork_accounts: Vec
, + /// Transition block for a change of difficulty params (currently just bound_divisor). + pub difficulty_hardfork_transition: u64, + /// Difficulty param after the difficulty transition. + pub difficulty_hardfork_bound_divisor: U256, + /// Block on which there is no additional difficulty from the exponential bomb. + pub bomb_defuse_transition: u64, } impl From for EthashParams { @@ -54,6 +62,7 @@ impl From for EthashParams { gas_limit_bound_divisor: p.gas_limit_bound_divisor.into(), minimum_difficulty: p.minimum_difficulty.into(), difficulty_bound_divisor: p.difficulty_bound_divisor.into(), + difficulty_increment_divisor: p.difficulty_increment_divisor.map_or(10, Into::into), duration_limit: p.duration_limit.into(), block_reward: p.block_reward.into(), registrar: p.registrar.map_or_else(Address::new, Into::into), @@ -61,6 +70,9 @@ impl From for EthashParams { dao_hardfork_transition: p.dao_hardfork_transition.map_or(0x7fffffffffffffff, Into::into), dao_hardfork_beneficiary: p.dao_hardfork_beneficiary.map_or_else(Address::new, Into::into), dao_hardfork_accounts: p.dao_hardfork_accounts.unwrap_or_else(Vec::new).into_iter().map(Into::into).collect(), + difficulty_hardfork_transition: p.difficulty_hardfork_transition.map_or(0x7fffffffffffffff, Into::into), + difficulty_hardfork_bound_divisor: p.difficulty_hardfork_bound_divisor.map_or(p.difficulty_bound_divisor.into(), Into::into), + bomb_defuse_transition: p.bomb_defuse_transition.map_or(0x7fffffffffffffff, Into::into), } } } @@ -168,6 +180,10 @@ impl Engine for Ethash { fields.state.add_balance(u.author(), &(reward * U256::from(8 + u.number() - current_number) / U256::from(8))); } + // Commit state so that we can actually figure out the state root. + if let Err(e) = fields.state.commit() { + warn!("Encountered error on state commit: {}", e); + } } fn verify_block_basic(&self, header: &Header, _block: Option<&[u8]>) -> result::Result<(), Error> { @@ -217,6 +233,7 @@ impl Engine for Ethash { let result = self.pow.compute_light(header.number() as u64, &Ethash::to_ethash(header.bare_hash()), header.nonce().low_u64()); let mix = Ethash::from_ethash(result.mix_hash); let difficulty = Ethash::boundary_to_difficulty(&Ethash::from_ethash(result.value)); + trace!(target: "miner", "num: {}, seed: {}, h: {}, non: {}, mix: {}, res: {}" , header.number() as u64, Ethash::from_ethash(slow_get_seedhash(header.number() as u64)), header.bare_hash(), header.nonce().low_u64(), Ethash::from_ethash(result.mix_hash), Ethash::from_ethash(result.value)); if mix != header.mix_hash() { return Err(From::from(BlockError::MismatchedH256SealElement(Mismatch { expected: mix, found: header.mix_hash() }))); } @@ -267,7 +284,11 @@ impl Ethash { } let min_difficulty = self.ethash_params.minimum_difficulty; - let difficulty_bound_divisor = self.ethash_params.difficulty_bound_divisor; + let difficulty_hardfork = header.number() >= self.ethash_params.difficulty_hardfork_transition; + let difficulty_bound_divisor = match difficulty_hardfork { + true => self.ethash_params.difficulty_hardfork_bound_divisor, + false => self.ethash_params.difficulty_bound_divisor, + }; let duration_limit = self.ethash_params.duration_limit; let frontier_limit = self.ethash_params.frontier_compatibility_mode_limit; @@ -281,17 +302,19 @@ impl Ethash { else { trace!(target: "ethash", "Calculating difficulty parent.difficulty={}, header.timestamp={}, parent.timestamp={}", parent.difficulty(), header.timestamp(), parent.timestamp()); //block_diff = parent_diff + parent_diff // 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99) - let diff_inc = (header.timestamp() - parent.timestamp()) / 10; + let diff_inc = (header.timestamp() - parent.timestamp()) / self.ethash_params.difficulty_increment_divisor; if diff_inc <= 1 { - parent.difficulty().clone() + parent.difficulty().clone() / From::from(2048) * From::from(1 - diff_inc) + parent.difficulty().clone() + parent.difficulty().clone() / From::from(difficulty_bound_divisor) * From::from(1 - diff_inc) } else { - parent.difficulty().clone() - parent.difficulty().clone() / From::from(2048) * From::from(min(diff_inc - 1, 99)) + parent.difficulty().clone() - parent.difficulty().clone() / From::from(difficulty_bound_divisor) * From::from(min(diff_inc - 1, 99)) } }; target = max(min_difficulty, target); - let period = ((parent.number() + 1) / EXP_DIFF_PERIOD) as usize; - if period > 1 { - target = max(min_difficulty, target + (U256::from(1) << (period - 2))); + if header.number() < self.ethash_params.bomb_defuse_transition { + let period = ((parent.number() + 1) / EXP_DIFF_PERIOD) as usize; + if period > 1 { + target = max(min_difficulty, target + (U256::from(1) << (period - 2))); + } } target } @@ -355,9 +378,9 @@ mod tests { let spec = new_morden(); let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let b = b.close(); @@ -369,9 +392,9 @@ mod tests { let spec = new_morden(); let engine = &*spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let last_hashes = Arc::new(vec![genesis_header.hash()]); let mut b = OpenBlock::new(engine, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap(); let mut uncle = Header::new(); diff --git a/ethcore/src/ethereum/mod.rs b/ethcore/src/ethereum/mod.rs index 6d46d5551..6d4502d2d 100644 --- a/ethcore/src/ethereum/mod.rs +++ b/ethcore/src/ethereum/mod.rs @@ -42,6 +42,9 @@ pub fn new_frontier() -> Spec { load(include_bytes!("../../res/ethereum/frontier /// Create a new Frontier mainnet chain spec without the DAO hardfork. pub fn new_classic() -> Spec { load(include_bytes!("../../res/ethereum/classic.json")) } +/// Create a new Frontier mainnet chain spec without the DAO hardfork. +pub fn new_expanse() -> Spec { load(include_bytes!("../../res/ethereum/expanse.json")) } + /// Create a new Frontier chain spec as though it never changes to Homestead. pub fn new_frontier_test() -> Spec { load(include_bytes!("../../res/ethereum/frontier_test.json")) } @@ -69,9 +72,9 @@ mod tests { let spec = new_morden(); let engine = &spec.engine; let genesis_header = spec.genesis_header(); - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + spec.ensure_db_good(&mut db).unwrap(); let s = State::from_existing(db, genesis_header.state_root().clone(), engine.account_start_nonce(), Default::default()).unwrap(); assert_eq!(s.balance(&"0000000000000000000000000000000000000001".into()), 1u64.into()); assert_eq!(s.balance(&"0000000000000000000000000000000000000002".into()), 1u64.into()); diff --git a/ethcore/src/evm/jit.rs b/ethcore/src/evm/jit.rs index c62f87ab7..3487d9a59 100644 --- a/ethcore/src/evm/jit.rs +++ b/ethcore/src/evm/jit.rs @@ -196,6 +196,7 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> { receive_address: *const evmjit::H256, code_address: *const evmjit::H256, transfer_value: *const evmjit::I256, + // We are ignoring apparent value - it's handled in externalities. _apparent_value: *const evmjit::I256, in_beg: *const u8, in_size: u64, @@ -208,12 +209,13 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> { let sender_address = unsafe { Address::from_jit(&*sender_address) }; let receive_address = unsafe { Address::from_jit(&*receive_address) }; let code_address = unsafe { Address::from_jit(&*code_address) }; - // TODO Is it always safe in case of DELEGATE_CALL? let transfer_value = unsafe { U256::from_jit(&*transfer_value) }; - let value = Some(transfer_value); // receive address and code address are the same in normal calls let is_callcode = receive_address != code_address; + let is_delegatecall = is_callcode && sender_address != receive_address; + + let value = if is_delegatecall { None } else { Some(transfer_value) }; if !is_callcode && !self.ext.exists(&code_address) { gas_cost = gas_cost + U256::from(self.ext.schedule().call_new_account_gas); @@ -242,10 +244,10 @@ impl<'a> evmjit::Ext for ExtAdapter<'a> { } } - // TODO [ToDr] Any way to detect DelegateCall? - let call_type = match is_callcode { - true => CallType::CallCode, - false => CallType::Call, + let call_type = match (is_callcode, is_delegatecall) { + (_, true) => CallType::DelegateCall, + (true, false) => CallType::CallCode, + (false, false) => CallType::Call, }; match self.ext.call( diff --git a/ethcore/src/evm/tests.rs b/ethcore/src/evm/tests.rs index ec217b6c5..89a4c4ba9 100644 --- a/ethcore/src/evm/tests.rs +++ b/ethcore/src/evm/tests.rs @@ -141,7 +141,7 @@ impl Ext for FakeExt { } fn extcodesize(&self, address: &Address) -> usize { - self.codes.get(address).map(|v| v.len()).unwrap_or(0) + self.codes.get(address).map_or(0, |c| c.len()) } fn log(&mut self, topics: Vec, data: &[u8]) { diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index 8f8b534ee..6a7e4672a 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -589,8 +589,11 @@ mod tests { assert_eq!(substate.contracts_created.len(), 0); } - evm_test!{test_call_to_create: test_call_to_create_jit, test_call_to_create_int} - fn test_call_to_create(factory: Factory) { + #[test] + // Tracing is not suported in JIT + fn test_call_to_create() { + let factory = Factory::new(VMType::Interpreter); + // code: // // 7c 601080600c6000396000f3006000355415600957005b60203560003555 - push 29 bytes? @@ -712,8 +715,10 @@ mod tests { assert_eq!(vm_tracer.drain().unwrap(), expected_vm_trace); } - evm_test!{test_create_contract: test_create_contract_jit, test_create_contract_int} - fn test_create_contract(factory: Factory) { + #[test] + fn test_create_contract() { + // Tracing is not supported in JIT + let factory = Factory::new(VMType::Interpreter); // code: // // 60 10 - push 16 diff --git a/ethcore/src/externalities.rs b/ethcore/src/externalities.rs index 7395522c3..82b946ffc 100644 --- a/ethcore/src/externalities.rs +++ b/ethcore/src/externalities.rs @@ -209,7 +209,6 @@ impl<'a, T, V> Ext for Externalities<'a, T, V> where T: 'a + Tracer, V: 'a + VMT self.state.code_size(address).unwrap_or(0) } - #[cfg_attr(feature="dev", allow(match_ref_pats))] fn ret(mut self, gas: &U256, data: &[u8]) -> evm::Result where Self: Sized { diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index 04581fac9..7d86cfd61 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -295,6 +295,12 @@ impl Encodable for Header { } } +impl HeapSizeOf for Header { + fn heap_size_of_children(&self) -> usize { + self.extra_data.heap_size_of_children() + self.seal.heap_size_of_children() + } +} + #[cfg(test)] mod tests { use rustc_serialize::hex::FromHex; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 5344381f0..cc398a2f5 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -110,6 +110,7 @@ extern crate lazy_static; extern crate heapsize; #[macro_use] extern crate ethcore_ipc as ipc; +extern crate lru_cache; #[cfg(feature = "jit" )] extern crate evmjit; @@ -119,7 +120,6 @@ pub extern crate ethstore; pub mod account_provider; pub mod engines; pub mod block; -pub mod block_queue; pub mod client; pub mod error; pub mod ethereum; @@ -143,6 +143,7 @@ mod basic_types; mod env_info; mod pod_account; mod state; +mod state_db; mod account_db; mod builtin; mod executive; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 182234280..2b0585db7 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -864,15 +864,24 @@ impl MinerService for Miner { } fn submit_seal(&self, chain: &MiningBlockChainClient, pow_hash: H256, seal: Vec) -> Result<(), Error> { - let result = if let Some(b) = self.sealing_work.lock().queue.get_used_if(if self.options.enable_resubmission { GetAction::Clone } else { GetAction::Take }, |b| &b.hash() == &pow_hash) { - b.lock().try_seal(&*self.engine, seal).or_else(|_| { - warn!(target: "miner", "Mined solution rejected: Invalid."); - Err(Error::PowInvalid) - }) - } else { - warn!(target: "miner", "Mined solution rejected: Block unknown or out of date."); - Err(Error::PowHashInvalid) - }; + let result = + if let Some(b) = self.sealing_work.lock().queue.get_used_if( + if self.options.enable_resubmission { + GetAction::Clone + } else { + GetAction::Take + }, + |b| &b.hash() == &pow_hash + ) { + trace!(target: "miner", "Sealing block {}={}={} with seal {:?}", pow_hash, b.hash(), b.header().bare_hash(), seal); + b.lock().try_seal(&*self.engine, seal).or_else(|(e, _)| { + warn!(target: "miner", "Mined solution rejected: {}", e); + Err(Error::PowInvalid) + }) + } else { + warn!(target: "miner", "Mined solution rejected: Block unknown or out of date."); + Err(Error::PowHashInvalid) + }; result.and_then(|sealed| { let n = sealed.header().number(); let h = sealed.header().hash(); @@ -915,7 +924,7 @@ impl MinerService for Miner { out_of_chain.for_each(|txs| { let mut transaction_queue = self.transaction_queue.lock(); let _ = self.add_transactions_to_queue( - chain, txs, TransactionOrigin::External, &mut transaction_queue + chain, txs, TransactionOrigin::RetractedBlock, &mut transaction_queue ); }); } diff --git a/ethcore/src/miner/transaction_queue.rs b/ethcore/src/miner/transaction_queue.rs index 7db65eacb..75a66358a 100644 --- a/ethcore/src/miner/transaction_queue.rs +++ b/ethcore/src/miner/transaction_queue.rs @@ -98,6 +98,8 @@ pub enum TransactionOrigin { Local, /// External transaction received from network External, + /// Transactions from retracted blocks + RetractedBlock, } impl PartialOrd for TransactionOrigin { @@ -112,10 +114,11 @@ impl Ord for TransactionOrigin { return Ordering::Equal; } - if *self == TransactionOrigin::Local { - Ordering::Less - } else { - Ordering::Greater + match (*self, *other) { + (TransactionOrigin::RetractedBlock, _) => Ordering::Less, + (_, TransactionOrigin::RetractedBlock) => Ordering::Greater, + (TransactionOrigin::Local, _) => Ordering::Less, + _ => Ordering::Greater, } } } @@ -711,7 +714,10 @@ impl TransactionQueue { let order = self.current.drop(sender, &k).expect("iterating over a collection that has been retrieved above; qed"); if k >= current_nonce { - self.future.insert(*sender, k, order.update_height(k, current_nonce)); + let order = order.update_height(k, current_nonce); + if let Some(old) = self.future.insert(*sender, k, order.clone()) { + Self::replace_orders(*sender, k, old, order, &mut self.future, &mut self.by_hash); + } } else { trace!(target: "txqueue", "Removing old transaction: {:?} (nonce: {} < {})", order.hash, k, current_nonce); self.by_hash.remove(&order.hash).expect("All transactions in `future` are also in `by_hash`"); @@ -776,7 +782,9 @@ impl TransactionQueue { self.future.by_gas_price.remove(&order.gas_price, &order.hash); // Put to current let order = order.update_height(current_nonce, first_nonce); - self.current.insert(address, current_nonce, order); + if let Some(old) = self.current.insert(address, current_nonce, order.clone()) { + Self::replace_orders(address, current_nonce, old, order, &mut self.current, &mut self.by_hash); + } update_last_nonce_to = Some(current_nonce); current_nonce = current_nonce + U256::one(); } @@ -810,47 +818,49 @@ impl TransactionQueue { let nonce = tx.nonce(); let hash = tx.hash(); - let next_nonce = self.last_nonces - .get(&address) - .cloned() - .map_or(state_nonce, |n| n + U256::one()); - // The transaction might be old, let's check that. // This has to be the first test, otherwise calculating // nonce height would result in overflow. if nonce < state_nonce { // Droping transaction - trace!(target: "txqueue", "Dropping old transaction: {:?} (nonce: {} < {})", tx.hash(), nonce, next_nonce); + trace!(target: "txqueue", "Dropping old transaction: {:?} (nonce: {} < {})", tx.hash(), nonce, state_nonce); return Err(TransactionError::Old); - } else if nonce > next_nonce { + } + + // Update nonces of transactions in future (remove old transactions) + self.update_future(&address, state_nonce); + // State nonce could be updated. Maybe there are some more items waiting in future? + self.move_matching_future_to_current(address, state_nonce, state_nonce); + // Check the next expected nonce (might be updated by move above) + let next_nonce = self.last_nonces + .get(&address) + .cloned() + .map_or(state_nonce, |n| n + U256::one()); + + // Future transaction + if nonce > next_nonce { // We have a gap - put to future. - // Update nonces of transactions in future (remove old transactions) - self.update_future(&address, state_nonce); // Insert transaction (or replace old one with lower gas price) try!(check_too_cheap(Self::replace_transaction(tx, state_nonce, &mut self.future, &mut self.by_hash))); - // Return an error if this transaction is not imported because of limit. - try!(check_if_removed(&address, &nonce, self.future.enforce_limit(&mut self.by_hash))); + // Enforce limit in Future + let removed = self.future.enforce_limit(&mut self.by_hash); + // Return an error if this transaction was not imported because of limit. + try!(check_if_removed(&address, &nonce, removed)); debug!(target: "txqueue", "Importing transaction to future: {:?}", hash); debug!(target: "txqueue", "status: {:?}", self.status()); return Ok(TransactionImportResult::Future); } + + // We might have filled a gap - move some more transactions from future + self.move_matching_future_to_current(address, nonce, state_nonce); + self.move_matching_future_to_current(address, nonce + U256::one(), state_nonce); + + // Replace transaction if any try!(check_too_cheap(Self::replace_transaction(tx, state_nonce, &mut self.current, &mut self.by_hash))); // Keep track of highest nonce stored in current let new_max = self.last_nonces.get(&address).map_or(nonce, |n| cmp::max(nonce, *n)); self.last_nonces.insert(address, new_max); - // Update nonces of transactions in future - self.update_future(&address, state_nonce); - // Maybe there are some more items waiting in future? - self.move_matching_future_to_current(address, nonce + U256::one(), state_nonce); - // There might be exactly the same transaction waiting in future - // same (sender, nonce), but above function would not move it. - if let Some(order) = self.future.drop(&address, &nonce) { - // Let's insert that transaction to current (if it has higher gas_price) - let future_tx = self.by_hash.remove(&order.hash).expect("All transactions in `future` are always in `by_hash`."); - // if transaction in `current` (then one we are importing) is replaced it means that it has to low gas_price - try!(check_too_cheap(!Self::replace_transaction(future_tx, state_nonce, &mut self.current, &mut self.by_hash))); - } // Also enforce the limit let removed = self.current.enforce_limit(&mut self.by_hash); @@ -895,24 +905,28 @@ impl TransactionQueue { if let Some(old) = set.insert(address, nonce, order.clone()) { - // There was already transaction in queue. Let's check which one should stay - let old_fee = old.gas_price; - let new_fee = order.gas_price; - if old_fee.cmp(&new_fee) == Ordering::Greater { - // Put back old transaction since it has greater priority (higher gas_price) - set.insert(address, nonce, old); - // and remove new one - by_hash.remove(&hash).expect("The hash has been just inserted and no other line is altering `by_hash`."); - false - } else { - // Make sure we remove old transaction entirely - by_hash.remove(&old.hash).expect("The hash is coming from `future` so it has to be in `by_hash`."); - true - } + Self::replace_orders(address, nonce, old, order, set, by_hash) } else { true } } + + fn replace_orders(address: Address, nonce: U256, old: TransactionOrder, order: TransactionOrder, set: &mut TransactionSet, by_hash: &mut HashMap) -> bool { + // There was already transaction in queue. Let's check which one should stay + let old_fee = old.gas_price; + let new_fee = order.gas_price; + if old_fee.cmp(&new_fee) == Ordering::Greater { + // Put back old transaction since it has greater priority (higher gas_price) + set.insert(address, nonce, old); + // and remove new one + by_hash.remove(&order.hash).expect("The hash has been just inserted and no other line is altering `by_hash`."); + false + } else { + // Make sure we remove old transaction entirely + by_hash.remove(&old.hash).expect("The hash is coming from `future` so it has to be in `by_hash`."); + true + } + } } fn check_too_cheap(is_in: bool) -> Result<(), TransactionError> { @@ -1014,6 +1028,17 @@ mod test { new_tx_pair_default(0.into(), 1.into()) } + #[test] + fn test_ordering() { + assert_eq!(TransactionOrigin::Local.cmp(&TransactionOrigin::External), Ordering::Less); + assert_eq!(TransactionOrigin::RetractedBlock.cmp(&TransactionOrigin::Local), Ordering::Less); + assert_eq!(TransactionOrigin::RetractedBlock.cmp(&TransactionOrigin::External), Ordering::Less); + + assert_eq!(TransactionOrigin::External.cmp(&TransactionOrigin::Local), Ordering::Greater); + assert_eq!(TransactionOrigin::Local.cmp(&TransactionOrigin::RetractedBlock), Ordering::Greater); + assert_eq!(TransactionOrigin::External.cmp(&TransactionOrigin::RetractedBlock), Ordering::Greater); + } + #[test] fn should_return_correct_nonces_when_dropped_because_of_limit() { // given @@ -1196,6 +1221,31 @@ mod test { assert_eq!(txq.top_transactions()[0], tx2); } + #[test] + fn should_move_all_transactions_from_future() { + // given + let mut txq = TransactionQueue::new(); + let (tx, tx2) = new_tx_pair_default(1.into(), 1.into()); + let prev_nonce = |a: &Address| AccountDetails{ nonce: default_account_details(a).nonce - U256::one(), balance: + !U256::zero() }; + + // First insert one transaction to future + let res = txq.add(tx.clone(), &prev_nonce, TransactionOrigin::External); + assert_eq!(res.unwrap(), TransactionImportResult::Future); + assert_eq!(txq.status().future, 1); + + // now import second transaction to current + let res = txq.add(tx2.clone(), &default_account_details, TransactionOrigin::External); + + // then + assert_eq!(res.unwrap(), TransactionImportResult::Current); + assert_eq!(txq.status().pending, 2); + assert_eq!(txq.status().future, 0); + assert_eq!(txq.current.by_priority.len(), 2); + assert_eq!(txq.current.by_address.len(), 2); + assert_eq!(txq.top_transactions()[0], tx); + assert_eq!(txq.top_transactions()[1], tx2); + } #[test] fn should_import_tx() { @@ -1375,6 +1425,27 @@ mod test { assert_eq!(top.len(), 2); } + #[test] + fn should_prioritize_reimported_transactions_within_same_nonce_height() { + // given + let mut txq = TransactionQueue::new(); + let tx = new_tx_default(); + // the second one has same nonce but higher `gas_price` + let (_, tx2) = new_similar_tx_pair(); + + // when + // first insert local one with higher gas price + txq.add(tx2.clone(), &default_account_details, TransactionOrigin::Local).unwrap(); + // then the one with lower gas price, but from retracted block + txq.add(tx.clone(), &default_account_details, TransactionOrigin::RetractedBlock).unwrap(); + + // then + let top = txq.top_transactions(); + assert_eq!(top[0], tx); // retracted should be first + assert_eq!(top[1], tx2); + assert_eq!(top.len(), 2); + } + #[test] fn should_not_prioritize_local_transactions_with_different_nonce_height() { // given diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 703d61742..a92e03ebc 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -48,7 +48,7 @@ impl PodAccount { PodAccount { balance: *acc.balance(), nonce: *acc.nonce(), - storage: acc.storage_overlay().iter().fold(BTreeMap::new(), |mut m, (k, &(_, ref v))| {m.insert(k.clone(), v.clone()); m}), + storage: acc.storage_changes().iter().fold(BTreeMap::new(), |mut m, (k, v)| {m.insert(k.clone(), v.clone()); m}), code: acc.code().map(|x| x.to_vec()), } } diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 9fa126cc7..b2dd18cd5 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -80,7 +80,13 @@ impl ClientService { } let mut db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); - db_config.cache_size = config.db_cache_size; + + // give all rocksdb cache to state column; everything else has its + // own caches. + if let Some(size) = config.db_cache_size { + db_config.set_cache(::db::COL_STATE, size); + } + db_config.compaction = config.db_compaction.compaction_profile(); db_config.wal = config.db_wal; @@ -90,7 +96,7 @@ impl ClientService { let snapshot_params = SnapServiceParams { engine: spec.engine.clone(), genesis_block: spec.genesis_block(), - db_config: db_config, + db_config: db_config.clone(), pruning: pruning, channel: io_service.channel(), snapshot_root: snapshot_path.into(), diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 8cfc4c96b..bc1faea3f 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -205,7 +205,7 @@ impl Account { #[cfg(test)] mod tests { use account_db::{AccountDB, AccountDBMut}; - use tests::helpers::get_temp_journal_db; + use tests::helpers::get_temp_state_db; use snapshot::tests::helpers::fill_storage; use util::sha3::{SHA3_EMPTY, SHA3_NULL_RLP}; @@ -218,8 +218,7 @@ mod tests { #[test] fn encoding_basic() { - let mut db = get_temp_journal_db(); - let mut db = &mut **db; + let mut db = get_temp_state_db(); let addr = Address::random(); let account = Account { @@ -239,8 +238,7 @@ mod tests { #[test] fn encoding_storage() { - let mut db = get_temp_journal_db(); - let mut db = &mut **db; + let mut db = get_temp_state_db(); let addr = Address::random(); let account = { @@ -265,8 +263,7 @@ mod tests { #[test] fn encoding_code() { - let mut db = get_temp_journal_db(); - let mut db = &mut **db; + let mut db = get_temp_state_db(); let addr1 = Address::random(); let addr2 = Address::random(); diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index c24849cba..d86f7ec46 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -20,6 +20,7 @@ use common::*; use engines::{Engine, NullEngine, InstantSeal, BasicAuthority, Tendermint}; use pod_state::*; use account_db::*; +use state_db::StateDB; use super::genesis::Genesis; use super::seal::Generic as GenericSeal; use ethereum; @@ -36,6 +37,8 @@ pub struct CommonParams { pub maximum_extra_data_size: usize, /// Network id. pub network_id: U256, + /// Main subprotocol name. + pub subprotocol_name: String, /// Minimum gas limit. pub min_gas_limit: U256, /// Fork block to check. @@ -48,6 +51,7 @@ impl From for CommonParams { account_start_nonce: p.account_start_nonce.into(), maximum_extra_data_size: p.maximum_extra_data_size.into(), network_id: p.network_id.into(), + subprotocol_name: p.subprotocol_name.unwrap_or_else(|| "eth".to_owned()), min_gas_limit: p.min_gas_limit.into(), fork_block: if let (Some(n), Some(h)) = (p.fork_block, p.fork_hash) { Some((n.into(), h.into())) } else { None }, } @@ -156,6 +160,9 @@ impl Spec { /// Get the configured Network ID. pub fn network_id(&self) -> U256 { self.params.network_id } + /// Get the configured Network ID. + pub fn subprotocol_name(&self) -> String { self.params.subprotocol_name.clone() } + /// Get the configured network fork block. pub fn fork_block(&self) -> Option<(BlockNumber, H256)> { self.params.fork_block } @@ -169,7 +176,7 @@ impl Spec { header.set_transactions_root(self.transactions_root.clone()); header.set_uncles_hash(RlpStream::new_list(0).out().sha3()); header.set_extra_data(self.extra_data.clone()); - header.set_state_root(self.state_root().clone()); + header.set_state_root(self.state_root()); header.set_receipts_root(self.receipts_root.clone()); header.set_log_bloom(H2048::new().clone()); header.set_gas_used(self.gas_used.clone()); @@ -184,6 +191,7 @@ impl Spec { let r = Rlp::new(&seal); (0..self.seal_fields).map(|i| r.at(i).as_raw().to_vec()).collect() }); + trace!(target: "spec", "Header hash is {}", header.hash()); header } @@ -227,19 +235,22 @@ impl Spec { } /// Ensure that the given state DB has the trie nodes in for the genesis state. - pub fn ensure_db_good(&self, db: &mut HashDB) -> Result> { - if !db.contains(&self.state_root()) { + pub fn ensure_db_good(&self, db: &mut StateDB) -> Result> { + if !db.as_hashdb().contains(&self.state_root()) { + trace!(target: "spec", "ensure_db_good: Fresh database? Cannot find state root {}", self.state_root()); let mut root = H256::new(); + { - let mut t = SecTrieDBMut::new(db, &mut root); + let mut t = SecTrieDBMut::new(db.as_hashdb_mut(), &mut root); for (address, account) in self.genesis_state.get().iter() { try!(t.insert(&**address, &account.rlp())); } } + trace!(target: "spec", "ensure_db_good: Populated sec trie; root is {}", root); for (address, account) in self.genesis_state.get().iter() { - account.insert_additional(&mut AccountDBMut::new(db, address)); + account.insert_additional(&mut AccountDBMut::new(db.as_hashdb_mut(), address)); } - assert!(db.contains(&self.state_root())); + assert!(db.as_hashdb().contains(&self.state_root())); Ok(true) } else { Ok(false) } } diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index cdd430290..07478220a 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -20,11 +20,13 @@ use std::collections::hash_map::Entry; use util::*; use pod_account::*; use rlp::*; +use lru_cache::LruCache; -use std::cell::{Ref, RefCell, Cell}; +use std::cell::{RefCell, Cell}; + +const STORAGE_CACHE_ITEMS: usize = 4096; /// Single account in the system. -#[derive(Clone)] pub struct Account { // Balance of the account. balance: U256, @@ -32,10 +34,16 @@ pub struct Account { nonce: U256, // Trie-backed storage. storage_root: H256, - // Overlay on trie-backed storage - tuple is (, ). - storage_overlay: RefCell>, + // LRU Cache of the trie-backed storage. + // This is limited to `STORAGE_CACHE_ITEMS` recent queries + storage_cache: RefCell>, + // Modified storage. Accumulates changes to storage made in `set_storage` + // Takes precedence over `storage_cache`. + storage_changes: HashMap, // Code hash of the account. If None, means that it's a contract whose code has not yet been set. code_hash: Option, + // Size of the accoun code. + code_size: Option, // Code cache of the account. code_cache: Bytes, // Account is new or has been modified @@ -52,23 +60,31 @@ impl Account { balance: balance, nonce: nonce, storage_root: SHA3_NULL_RLP, - storage_overlay: RefCell::new(storage.into_iter().map(|(k, v)| (k, (Filth::Dirty, v))).collect()), + storage_cache: Self::empty_storage_cache(), + storage_changes: storage, code_hash: Some(code.sha3()), + code_size: Some(code.len()), code_cache: code, filth: Filth::Dirty, address_hash: Cell::new(None), } } + fn empty_storage_cache() -> RefCell> { + RefCell::new(LruCache::new(STORAGE_CACHE_ITEMS)) + } + /// General constructor. pub fn from_pod(pod: PodAccount) -> Account { Account { balance: pod.balance, nonce: pod.nonce, storage_root: SHA3_NULL_RLP, - storage_overlay: RefCell::new(pod.storage.into_iter().map(|(k, v)| (k, (Filth::Dirty, v))).collect()), + storage_cache: Self::empty_storage_cache(), + storage_changes: pod.storage.into_iter().collect(), code_hash: pod.code.as_ref().map(|c| c.sha3()), - code_cache: pod.code.as_ref().map_or_else(|| { warn!("POD account with unknown code is being created! Assuming no code."); vec![] }, |c| c.clone()), + code_size: Some(pod.code.as_ref().map_or(0, |c| c.len())), + code_cache: pod.code.map_or_else(|| { warn!("POD account with unknown code is being created! Assuming no code."); vec![] }, |c| c), filth: Filth::Dirty, address_hash: Cell::new(None), } @@ -80,9 +96,11 @@ impl Account { balance: balance, nonce: nonce, storage_root: SHA3_NULL_RLP, - storage_overlay: RefCell::new(HashMap::new()), + storage_cache: Self::empty_storage_cache(), + storage_changes: HashMap::new(), code_hash: Some(SHA3_EMPTY), code_cache: vec![], + code_size: Some(0), filth: Filth::Dirty, address_hash: Cell::new(None), } @@ -95,9 +113,11 @@ impl Account { nonce: r.val_at(0), balance: r.val_at(1), storage_root: r.val_at(2), - storage_overlay: RefCell::new(HashMap::new()), + storage_cache: Self::empty_storage_cache(), + storage_changes: HashMap::new(), code_hash: Some(r.val_at(3)), code_cache: vec![], + code_size: None, filth: Filth::Clean, address_hash: Cell::new(None), } @@ -110,9 +130,11 @@ impl Account { balance: balance, nonce: nonce, storage_root: SHA3_NULL_RLP, - storage_overlay: RefCell::new(HashMap::new()), + storage_cache: Self::empty_storage_cache(), + storage_changes: HashMap::new(), code_hash: None, code_cache: vec![], + code_size: None, filth: Filth::Dirty, address_hash: Cell::new(None), } @@ -123,44 +145,62 @@ impl Account { pub fn init_code(&mut self, code: Bytes) { assert!(self.code_hash.is_none()); self.code_cache = code; + self.code_size = Some(self.code_cache.len()); self.filth = Filth::Dirty; } /// Reset this account's code to the given code. pub fn reset_code(&mut self, code: Bytes) { self.code_hash = None; + self.code_size = Some(0); self.init_code(code); } /// Set (and cache) the contents of the trie's storage at `key` to `value`. pub fn set_storage(&mut self, key: H256, value: H256) { - match self.storage_overlay.borrow_mut().entry(key) { - Entry::Occupied(ref mut entry) if entry.get().1 != value => { - entry.insert((Filth::Dirty, value)); + match self.storage_changes.entry(key) { + Entry::Occupied(ref mut entry) if entry.get() != &value => { + entry.insert(value); self.filth = Filth::Dirty; }, Entry::Vacant(entry) => { - entry.insert((Filth::Dirty, value)); + entry.insert(value); self.filth = Filth::Dirty; }, - _ => (), + _ => {}, } } /// Get (and cache) the contents of the trie's storage at `key`. + /// Takes modifed storage into account. pub fn storage_at(&self, db: &HashDB, key: &H256) -> H256 { - self.storage_overlay.borrow_mut().entry(key.clone()).or_insert_with(||{ - let db = SecTrieDB::new(db, &self.storage_root) - .expect("Account storage_root initially set to zero (valid) and only altered by SecTrieDBMut. \ - SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \ - using it will not fail."); + if let Some(value) = self.cached_storage_at(key) { + return value; + } + let db = SecTrieDB::new(db, &self.storage_root) + .expect("Account storage_root initially set to zero (valid) and only altered by SecTrieDBMut. \ + SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \ + using it will not fail."); - let item: U256 = match db.get(key){ - Ok(x) => x.map_or_else(U256::zero, decode), - Err(e) => panic!("Encountered potential DB corruption: {}", e), - }; - (Filth::Clean, item.into()) - }).1.clone() + let item: U256 = match db.get(key){ + Ok(x) => x.map_or_else(U256::zero, decode), + Err(e) => panic!("Encountered potential DB corruption: {}", e), + }; + let value: H256 = item.into(); + self.storage_cache.borrow_mut().insert(key.clone(), value.clone()); + value + } + + /// Get cached storage value if any. Returns `None` if the + /// key is not in the cache. + pub fn cached_storage_at(&self, key: &H256) -> Option { + if let Some(value) = self.storage_changes.get(key) { + return Some(value.clone()) + } + if let Some(value) = self.storage_cache.borrow_mut().get_mut(key) { + return Some(value.clone()) + } + None } /// return the balance associated with this account. @@ -196,6 +236,12 @@ impl Account { } } + /// returns the account's code size. If `None` then the code cache or code size cache isn't available - + /// get someone who knows to call `note_code`. + pub fn code_size(&self) -> Option { + self.code_size.clone() + } + #[cfg(test)] /// Provide a byte array which hashes to the `code_hash`. returns the hash as a result. pub fn note_code(&mut self, code: Bytes) -> Result<(), H256> { @@ -203,6 +249,7 @@ impl Account { match self.code_hash { Some(ref i) if h == *i => { self.code_cache = code; + self.code_size = Some(self.code_cache.len()); Ok(()) }, _ => Err(h) @@ -216,11 +263,12 @@ impl Account { /// Is this a new or modified account? pub fn is_dirty(&self) -> bool { - self.filth == Filth::Dirty + self.filth == Filth::Dirty || !self.storage_is_clean() } /// Mark account as clean. pub fn set_clean(&mut self) { + assert!(self.storage_is_clean()); self.filth = Filth::Clean } @@ -231,7 +279,31 @@ impl Account { self.is_cached() || match self.code_hash { Some(ref h) => match db.get(h) { - Some(x) => { self.code_cache = x.to_vec(); true }, + Some(x) => { + self.code_cache = x.to_vec(); + self.code_size = Some(x.len()); + true + }, + _ => { + warn!("Failed reverse get of {}", h); + false + }, + }, + _ => false, + } + } + + /// Provide a database to get `code_size`. Should not be called if it is a contract without code. + pub fn cache_code_size(&mut self, db: &HashDB) -> bool { + // TODO: fill out self.code_cache; + trace!("Account::cache_code_size: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); + self.code_size.is_some() || + match self.code_hash { + Some(ref h) if h != &SHA3_EMPTY => match db.get(h) { + Some(x) => { + self.code_size = Some(x.len()); + true + }, _ => { warn!("Failed reverse get of {}", h); false @@ -241,16 +313,15 @@ impl Account { } } - #[cfg(test)] /// Determine whether there are any un-`commit()`-ed storage-setting operations. - pub fn storage_is_clean(&self) -> bool { self.storage_overlay.borrow().iter().find(|&(_, &(f, _))| f == Filth::Dirty).is_none() } + pub fn storage_is_clean(&self) -> bool { self.storage_changes.is_empty() } #[cfg(test)] /// return the storage root associated with this account or None if it has been altered via the overlay. pub fn storage_root(&self) -> Option<&H256> { if self.storage_is_clean() {Some(&self.storage_root)} else {None} } /// return the storage overlay. - pub fn storage_overlay(&self) -> Ref> { self.storage_overlay.borrow() } + pub fn storage_changes(&self) -> &HashMap { &self.storage_changes } /// Increment the nonce of the account by one. pub fn inc_nonce(&mut self) { @@ -276,26 +347,24 @@ impl Account { } } - /// Commit the `storage_overlay` to the backing DB and update `storage_root`. + /// Commit the `storage_changes` to the backing DB and update `storage_root`. pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) { let mut t = trie_factory.from_existing(db, &mut self.storage_root) .expect("Account storage_root initially set to zero (valid) and only altered by SecTrieDBMut. \ SecTrieDBMut would not set it to an invalid state root. Therefore the root is valid and DB creation \ using it will not fail."); - for (k, &mut (ref mut f, ref mut v)) in self.storage_overlay.borrow_mut().iter_mut() { - if f == &Filth::Dirty { - // cast key and value to trait type, - // so we can call overloaded `to_bytes` method - let res = match v.is_zero() { - true => t.remove(k), - false => t.insert(k, &encode(&U256::from(&*v))), - }; + for (k, v) in self.storage_changes.drain() { + // cast key and value to trait type, + // so we can call overloaded `to_bytes` method + let res = match v.is_zero() { + true => t.remove(&k), + false => t.insert(&k, &encode(&U256::from(&*v))), + }; - if let Err(e) = res { - warn!("Encountered potential DB corruption: {}", e); - } - *f = Filth::Clean; + if let Err(e) = res { + warn!("Encountered potential DB corruption: {}", e); } + self.storage_cache.borrow_mut().insert(k, v); } } @@ -303,9 +372,13 @@ impl Account { pub fn commit_code(&mut self, db: &mut HashDB) { trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty()); match (self.code_hash.is_none(), self.code_cache.is_empty()) { - (true, true) => self.code_hash = Some(SHA3_EMPTY), + (true, true) => { + self.code_hash = Some(SHA3_EMPTY); + self.code_size = Some(0); + }, (true, false) => { self.code_hash = Some(db.insert(&self.code_cache)); + self.code_size = Some(self.code_cache.len()); }, (false, _) => {}, } @@ -317,9 +390,57 @@ impl Account { stream.append(&self.nonce); stream.append(&self.balance); stream.append(&self.storage_root); - stream.append(self.code_hash.as_ref().expect("Cannot form RLP of contract account without code.")); + stream.append(self.code_hash.as_ref().unwrap_or(&SHA3_EMPTY)); stream.out() } + + /// Clone basic account data + pub fn clone_basic(&self) -> Account { + Account { + balance: self.balance.clone(), + nonce: self.nonce.clone(), + storage_root: self.storage_root.clone(), + storage_cache: Self::empty_storage_cache(), + storage_changes: HashMap::new(), + code_hash: self.code_hash.clone(), + code_size: self.code_size.clone(), + code_cache: Bytes::new(), + filth: self.filth, + address_hash: self.address_hash.clone(), + } + } + + /// Clone account data and dirty storage keys + pub fn clone_dirty(&self) -> Account { + let mut account = self.clone_basic(); + account.storage_changes = self.storage_changes.clone(); + account.code_cache = self.code_cache.clone(); + account + } + + /// Clone account data, dirty storage keys and cached storage keys. + pub fn clone_all(&self) -> Account { + let mut account = self.clone_dirty(); + account.storage_cache = self.storage_cache.clone(); + account + } + + /// Replace self with the data from other account merging storage cache + pub fn merge_with(&mut self, other: Account) { + assert!(self.storage_is_clean()); + assert!(other.storage_is_clean()); + self.balance = other.balance; + self.nonce = other.nonce; + self.storage_root = other.storage_root; + self.code_hash = other.code_hash; + self.code_cache = other.code_cache; + self.code_size = other.code_size; + self.address_hash = other.address_hash; + let mut cache = self.storage_cache.borrow_mut(); + for (k, v) in other.storage_cache.into_inner().into_iter() { + cache.insert(k.clone() , v.clone()); //TODO: cloning should not be required here + } + } } impl fmt::Debug for Account { @@ -416,6 +537,7 @@ mod tests { let mut db = AccountDBMut::new(&mut db, &Address::new()); a.init_code(vec![0x55, 0x44, 0xffu8]); assert_eq!(a.code_hash(), SHA3_EMPTY); + assert_eq!(a.code_size(), Some(3)); a.commit_code(&mut db); assert_eq!(a.code_hash().hex(), "af231e631776a517ca23125370d542873eca1fb4d613ed9b5d5335a46ae5b7eb"); } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index e1299d1dc..79d7cba54 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -23,6 +23,7 @@ use trace::FlatTrace; use pod_account::*; use pod_state::{self, PodState}; use types::state_diff::StateDiff; +use state_db::StateDB; mod account; mod substate; @@ -41,23 +42,92 @@ pub struct ApplyOutcome { /// Result type for the execution ("application") of a transaction. pub type ApplyResult = Result; +#[derive(Debug)] +enum AccountEntry { + /// Contains account data. + Cached(Account), + /// Account has been deleted. + Killed, + /// Account does not exist. + Missing, +} + +impl AccountEntry { + fn is_dirty(&self) -> bool { + match *self { + AccountEntry::Cached(ref a) => a.is_dirty(), + AccountEntry::Killed => true, + AccountEntry::Missing => false, + } + } + + /// Clone dirty data into new `AccountEntry`. + /// Returns None if clean. + fn clone_dirty(&self) -> Option { + match *self { + AccountEntry::Cached(ref acc) if acc.is_dirty() => Some(AccountEntry::Cached(acc.clone_dirty())), + AccountEntry::Killed => Some(AccountEntry::Killed), + _ => None, + } + } + + /// Clone account entry data that needs to be saved in the snapshot. + /// This includes basic account information and all locally cached storage keys + fn clone_for_snapshot(&self) -> AccountEntry { + match *self { + AccountEntry::Cached(ref acc) => AccountEntry::Cached(acc.clone_all()), + AccountEntry::Killed => AccountEntry::Killed, + AccountEntry::Missing => AccountEntry::Missing, + } + } +} + /// Representation of the entire state of all accounts in the system. +/// +/// `State` can work together with `StateDB` to share account cache. +/// +/// Local cache contains changes made locally and changes accumulated +/// locally from previous commits. Global cache reflects the database +/// state and never contains any changes. +/// +/// Account data can be in the following cache states: +/// * In global but not local - something that was queried from the database, +/// but never modified +/// * In local but not global - something that was just added (e.g. new account) +/// * In both with the same value - something that was changed to a new value, +/// but changed back to a previous block in the same block (same State instance) +/// * In both with different values - something that was overwritten with a +/// new value. +/// +/// All read-only state queries check local cache/modifications first, +/// then global state cache. If data is not found in any of the caches +/// it is loaded from the DB to the local cache. +/// +/// Upon destruction all the local cache data merged into the global cache. +/// The merge might be rejected if current state is non-canonical. pub struct State { - db: Box, + db: StateDB, root: H256, - cache: RefCell>>, - snapshots: RefCell>>>>, + cache: RefCell>, + snapshots: RefCell>>>, account_start_nonce: U256, factories: Factories, } +#[derive(Copy, Clone)] +enum RequireCache { + None, + CodeSize, + Code, +} + const SEC_TRIE_DB_UNWRAP_STR: &'static str = "A state can only be created with valid root. Creating a SecTrieDB with a valid root will not fail. \ Therefore creating a SecTrieDB with this state's root will not fail."; impl State { /// Creates new state with empty state root #[cfg(test)] - pub fn new(mut db: Box, account_start_nonce: U256, factories: Factories) -> State { + pub fn new(mut db: StateDB, account_start_nonce: U256, factories: Factories) -> State { let mut root = H256::new(); { // init trie and reset root too null @@ -75,7 +145,7 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: Box, root: H256, account_start_nonce: U256, factories: Factories) -> Result { + pub fn from_existing(db: StateDB, root: H256, account_start_nonce: U256, factories: Factories) -> Result { if !db.as_hashdb().contains(&root) { return Err(TrieError::InvalidStateRoot(root)); } @@ -119,14 +189,21 @@ impl State { self.cache.borrow_mut().insert(k, v); }, None => { - self.cache.borrow_mut().remove(&k); + match self.cache.borrow_mut().entry(k) { + ::std::collections::hash_map::Entry::Occupied(e) => { + if e.get().is_dirty() { + e.remove(); + } + }, + _ => {} + } } } } } } - fn insert_cache(&self, address: &Address, account: Option) { + fn insert_cache(&self, address: &Address, account: AccountEntry) { if let Some(ref mut snapshot) = self.snapshots.borrow_mut().last_mut() { if !snapshot.contains_key(address) { snapshot.insert(address.clone(), self.cache.borrow_mut().insert(address.clone(), account)); @@ -139,13 +216,14 @@ impl State { fn note_cache(&self, address: &Address) { if let Some(ref mut snapshot) = self.snapshots.borrow_mut().last_mut() { if !snapshot.contains_key(address) { - snapshot.insert(address.clone(), self.cache.borrow().get(address).cloned()); + snapshot.insert(address.clone(), self.cache.borrow().get(address).map(AccountEntry::clone_for_snapshot)); } } } /// Destroy the current object and return root and database. - pub fn drop(self) -> (H256, Box) { + pub fn drop(mut self) -> (H256, StateDB) { + self.commit_cache(); (self.root, self.db) } @@ -157,50 +235,99 @@ impl State { /// Create a new contract at address `contract`. If there is already an account at the address /// it will have its code reset, ready for `init_code()`. pub fn new_contract(&mut self, contract: &Address, balance: U256) { - self.insert_cache(contract, Some(Account::new_contract(balance, self.account_start_nonce))); + self.insert_cache(contract, AccountEntry::Cached(Account::new_contract(balance, self.account_start_nonce))); } /// Remove an existing account. pub fn kill_account(&mut self, account: &Address) { - self.insert_cache(account, None); + self.insert_cache(account, AccountEntry::Killed); } /// Determine whether an account exists. pub fn exists(&self, a: &Address) -> bool { - self.ensure_cached(a, false, |a| a.is_some()) + self.ensure_cached(a, RequireCache::None, |a| a.is_some()) } /// Get the balance of account `a`. pub fn balance(&self, a: &Address) -> U256 { - self.ensure_cached(a, false, + self.ensure_cached(a, RequireCache::None, |a| a.as_ref().map_or(U256::zero(), |account| *account.balance())) } /// Get the nonce of account `a`. pub fn nonce(&self, a: &Address) -> U256 { - self.ensure_cached(a, false, + self.ensure_cached(a, RequireCache::None, |a| a.as_ref().map_or(self.account_start_nonce, |account| *account.nonce())) } /// Mutate storage of account `address` so that it is `value` for `key`. pub fn storage_at(&self, address: &Address, key: &H256) -> H256 { - self.ensure_cached(address, false, |a| a.as_ref().map_or(H256::new(), |a| { - let addr_hash = a.address_hash(address); - let db = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash); - a.storage_at(db.as_hashdb(), key) - })) + // Storage key search and update works like this: + // 1. If there's an entry for the account in the local cache check for the key and return it if found. + // 2. If there's an entry for the account in the global cache check for the key or load it into that account. + // 3. If account is missing in the global cache load it into the local cache and cache the key there. + + // check local cache first without updating + { + let local_cache = self.cache.borrow_mut(); + let mut local_account = None; + if let Some(maybe_acc) = local_cache.get(address) { + match *maybe_acc { + AccountEntry::Cached(ref account) => { + if let Some(value) = account.cached_storage_at(key) { + return value; + } else { + local_account = Some(maybe_acc); + } + }, + _ => return H256::new(), + } + } + // check the global cache and and cache storage key there if found, + // otherwise cache the account localy and cache storage key there. + if let Some(result) = self.db.get_cached(address, |acc| acc.map_or(H256::new(), |a| { + let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), a.address_hash(address)); + a.storage_at(account_db.as_hashdb(), key) + })) { + return result; + } + if let Some(ref mut acc) = local_account { + if let AccountEntry::Cached(ref account) = **acc { + let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(address)); + return account.storage_at(account_db.as_hashdb(), key) + } else { + return H256::new() + } + } + } + + // account is not found in the global cache, get from the DB and insert into local + let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); + let maybe_acc = match db.get(address) { + Ok(acc) => acc.map(Account::from_rlp), + Err(e) => panic!("Potential DB corruption encountered: {}", e), + }; + let r = maybe_acc.as_ref().map_or(H256::new(), |a| { + let account_db = self.factories.accountdb.readonly(self.db.as_hashdb(), a.address_hash(address)); + a.storage_at(account_db.as_hashdb(), key) + }); + match maybe_acc { + Some(account) => self.insert_cache(address, AccountEntry::Cached(account)), + None => self.insert_cache(address, AccountEntry::Missing), + } + r } - /// Get the code of account `a`. + /// Get accounts' code. pub fn code(&self, a: &Address) -> Option { - self.ensure_cached(a, true, - |a| a.as_ref().map_or(None, |a| a.code().map(|x| x.to_vec()))) + self.ensure_cached(a, RequireCache::Code, + |a| a.as_ref().map_or(None, |a| a.code().map(|x|x.to_vec()))) } - /// Get the code size of account `a`. + /// Get accounts' code size. pub fn code_size(&self, a: &Address) -> Option { - self.ensure_cached(a, true, - |a| a.as_ref().map_or(None, |a| a.code().map(|x| x.len()))) + self.ensure_cached(a, RequireCache::CodeSize, + |a| a.as_ref().and_then(|a| a.code_size())) } /// Add `incr` to the balance of account `a`. @@ -262,19 +389,19 @@ impl State { /// Commit accounts to SecTrieDBMut. This is similar to cpp-ethereum's dev::eth::commit. /// `accounts` is mutable because we may need to commit the code or storage and record that. #[cfg_attr(feature="dev", allow(match_ref_pats))] - pub fn commit_into( + fn commit_into( factories: &Factories, - db: &mut HashDB, + db: &mut StateDB, root: &mut H256, - accounts: &mut HashMap> + accounts: &mut HashMap ) -> Result<(), Error> { // first, commit the sub trees. // TODO: is this necessary or can we dispense with the `ref mut a` for just `a`? for (address, ref mut a) in accounts.iter_mut() { match a { - &mut&mut Some(ref mut account) if account.is_dirty() => { + &mut&mut AccountEntry::Cached(ref mut account) if account.is_dirty() => { let addr_hash = account.address_hash(address); - let mut account_db = factories.accountdb.create(db, addr_hash); + let mut account_db = factories.accountdb.create(db.as_hashdb_mut(), addr_hash); account.commit_storage(&factories.trie, account_db.as_hashdb_mut()); account.commit_code(account_db.as_hashdb_mut()); } @@ -283,15 +410,18 @@ impl State { } { - let mut trie = factories.trie.from_existing(db, root).unwrap(); + let mut trie = factories.trie.from_existing(db.as_hashdb_mut(), root).unwrap(); for (address, ref mut a) in accounts.iter_mut() { match **a { - Some(ref mut account) if account.is_dirty() => { + AccountEntry::Cached(ref mut account) if account.is_dirty() => { account.set_clean(); - try!(trie.insert(address, &account.rlp())) + try!(trie.insert(address, &account.rlp())); }, - None => try!(trie.remove(address)), - _ => (), + AccountEntry::Killed => { + try!(trie.remove(address)); + **a = AccountEntry::Missing; + }, + _ => {}, } } } @@ -299,10 +429,27 @@ impl State { Ok(()) } + fn commit_cache(&mut self) { + let mut addresses = self.cache.borrow_mut(); + for (address, a) in addresses.drain() { + match a { + AccountEntry::Cached(account) => { + if !account.is_dirty() { + self.db.cache_account(address, Some(account)); + } + }, + AccountEntry::Missing => { + self.db.cache_account(address, None); + }, + _ => {}, + } + } + } + /// Commits our cached account changes into the trie. pub fn commit(&mut self) -> Result<(), Error> { assert!(self.snapshots.borrow().is_empty()); - Self::commit_into(&self.factories, self.db.as_hashdb_mut(), &mut self.root, &mut *self.cache.borrow_mut()) + Self::commit_into(&self.factories, &mut self.db, &mut self.root, &mut *self.cache.borrow_mut()) } /// Clear state cache @@ -316,7 +463,7 @@ impl State { pub fn populate_from(&mut self, accounts: PodState) { assert!(self.snapshots.borrow().is_empty()); for (add, acc) in accounts.drain().into_iter() { - self.cache.borrow_mut().insert(add, Some(Account::from_pod(acc))); + self.cache.borrow_mut().insert(add, AccountEntry::Cached(Account::from_pod(acc))); } } @@ -326,7 +473,7 @@ impl State { // TODO: handle database rather than just the cache. // will need fat db. PodState::from(self.cache.borrow().iter().fold(BTreeMap::new(), |mut m, (add, opt)| { - if let Some(ref acc) = *opt { + if let AccountEntry::Cached(ref acc) = *opt { m.insert(add.clone(), PodAccount::from_account(acc)); } m @@ -335,7 +482,7 @@ impl State { fn query_pod(&mut self, query: &PodState) { for (address, pod_account) in query.get() { - self.ensure_cached(address, true, |a| { + self.ensure_cached(address, RequireCache::Code, |a| { if a.is_some() { for key in pod_account.storage.keys() { self.storage_at(address, key); @@ -354,28 +501,61 @@ impl State { pod_state::diff_pod(&state_pre.to_pod(), &pod_state_post) } - /// Ensure account `a` is in our cache of the trie DB and return a handle for getting it. - /// `require_code` requires that the code be cached, too. - fn ensure_cached<'a, F, U>(&'a self, a: &'a Address, require_code: bool, f: F) -> U - where F: FnOnce(&Option) -> U { - let have_key = self.cache.borrow().contains_key(a); - if !have_key { - let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); - let maybe_acc = match db.get(a) { - Ok(acc) => acc.map(Account::from_rlp), - Err(e) => panic!("Potential DB corruption encountered: {}", e), - }; - self.insert_cache(a, maybe_acc); - } - if require_code { - if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() { - let addr_hash = account.address_hash(a); - let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash); - account.cache_code(accountdb.as_hashdb()); + fn update_account_cache(require: RequireCache, account: &mut Account, db: &HashDB) { + match require { + RequireCache::None => {}, + RequireCache::Code => { + account.cache_code(db); + } + RequireCache::CodeSize => { + account.cache_code_size(db); } } + } - f(self.cache.borrow().get(a).unwrap()) + /// Check caches for required data + /// First searches for account in the local, then the shared cache. + /// Populates local cache if nothing found. + fn ensure_cached(&self, a: &Address, require: RequireCache, f: F) -> U + where F: Fn(Option<&Account>) -> U { + // check local cache first + if let Some(ref mut maybe_acc) = self.cache.borrow_mut().get_mut(a) { + if let AccountEntry::Cached(ref mut account) = **maybe_acc { + let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a)); + Self::update_account_cache(require, account, accountdb.as_hashdb()); + return f(Some(account)); + } + return f(None); + } + // check global cache + let result = self.db.get_cached(a, |mut acc| { + if let Some(ref mut account) = acc { + let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a)); + Self::update_account_cache(require, account, accountdb.as_hashdb()); + } + f(acc.map(|a| &*a)) + }); + match result { + Some(r) => r, + None => { + // not found in the global cache, get from the DB and insert into local + let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); + let mut maybe_acc = match db.get(a) { + Ok(acc) => acc.map(Account::from_rlp), + Err(e) => panic!("Potential DB corruption encountered: {}", e), + }; + if let Some(ref mut account) = maybe_acc.as_mut() { + let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), account.address_hash(a)); + Self::update_account_cache(require, account, accountdb.as_hashdb()); + } + let r = f(maybe_acc.as_ref()); + match maybe_acc { + Some(account) => self.insert_cache(a, AccountEntry::Cached(account)), + None => self.insert_cache(a, AccountEntry::Missing), + } + r + } + } } /// Pull account `a` in our cache from the trie DB. `require_code` requires that the code be cached, too. @@ -390,30 +570,40 @@ impl State { { let contains_key = self.cache.borrow().contains_key(a); if !contains_key { - let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); - let maybe_acc = match db.get(a) { - Ok(acc) => acc.map(Account::from_rlp), - Err(e) => panic!("Potential DB corruption encountered: {}", e), - }; - - self.insert_cache(a, maybe_acc); + match self.db.get_cached_account(a) { + Some(Some(acc)) => self.insert_cache(a, AccountEntry::Cached(acc)), + Some(None) => self.insert_cache(a, AccountEntry::Missing), + None => { + let db = self.factories.trie.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR); + let maybe_acc = match db.get(a) { + Ok(Some(acc)) => AccountEntry::Cached(Account::from_rlp(acc)), + Ok(None) => AccountEntry::Missing, + Err(e) => panic!("Potential DB corruption encountered: {}", e), + }; + self.insert_cache(a, maybe_acc); + } + } } else { self.note_cache(a); } match self.cache.borrow_mut().get_mut(a).unwrap() { - &mut Some(ref mut acc) => not_default(acc), - slot @ &mut None => *slot = Some(default()), + &mut AccountEntry::Cached(ref mut acc) => not_default(acc), + slot => *slot = AccountEntry::Cached(default()), } RefMut::map(self.cache.borrow_mut(), |c| { - let account = c.get_mut(a).unwrap().as_mut().unwrap(); - if require_code { - let addr_hash = account.address_hash(a); - let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash); - account.cache_code(accountdb.as_hashdb()); + match c.get_mut(a).unwrap() { + &mut AccountEntry::Cached(ref mut account) => { + if require_code { + let addr_hash = account.address_hash(a); + let accountdb = self.factories.accountdb.readonly(self.db.as_hashdb(), addr_hash); + account.cache_code(accountdb.as_hashdb()); + } + account + }, + _ => panic!("Required account must always exist; qed"), } - account }) } } @@ -427,17 +617,10 @@ impl fmt::Debug for State { impl Clone for State { fn clone(&self) -> State { let cache = { - let mut cache = HashMap::new(); + let mut cache: HashMap = HashMap::new(); for (key, val) in self.cache.borrow().iter() { - let key = key.clone(); - match *val { - Some(ref acc) if acc.is_dirty() => { - cache.insert(key, Some(acc.clone())); - }, - None => { - cache.insert(key, None); - }, - _ => {}, + if let Some(entry) = val.clone_dirty() { + cache.insert(key.clone(), entry); } } cache @@ -447,7 +630,7 @@ impl Clone for State { db: self.db.boxed_clone(), root: self.root.clone(), cache: RefCell::new(cache), - snapshots: RefCell::new(self.snapshots.borrow().clone()), + snapshots: RefCell::new(Vec::new()), account_start_nonce: self.account_start_nonce.clone(), factories: self.factories.clone(), } diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs new file mode 100644 index 000000000..af6780cdc --- /dev/null +++ b/ethcore/src/state_db.rs @@ -0,0 +1,161 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use lru_cache::LruCache; +use util::journaldb::JournalDB; +use util::hash::{H256}; +use util::hashdb::HashDB; +use util::{Arc, Address, DBTransaction, UtilError, Mutex}; +use state::Account; + +const STATE_CACHE_ITEMS: usize = 65536; + +struct AccountCache { + /// DB Account cache. `None` indicates that account is known to be missing. + accounts: LruCache>, +} + +/// State database abstraction. +/// Manages shared global state cache. +/// A clone of `StateDB` may be created as canonical or not. +/// For canonical clones cache changes are accumulated and applied +/// on commit. +/// For non-canonical clones cache is cleared on commit. +pub struct StateDB { + db: Box, + account_cache: Arc>, + cache_overlay: Vec<(Address, Option)>, + is_canon: bool, +} + +impl StateDB { + /// Create a new instance wrapping `JournalDB` + pub fn new(db: Box) -> StateDB { + StateDB { + db: db, + account_cache: Arc::new(Mutex::new(AccountCache { accounts: LruCache::new(STATE_CACHE_ITEMS) })), + cache_overlay: Vec::new(), + is_canon: false, + } + } + + /// Commit all recent insert operations and canonical historical commits' removals from the + /// old era to the backing database, reverting any non-canonical historical commit's inserts. + pub fn commit(&mut self, batch: &mut DBTransaction, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let records = try!(self.db.commit(batch, now, id, end)); + if self.is_canon { + self.commit_cache(); + } else { + self.clear_cache(); + } + Ok(records) + } + + /// Returns an interface to HashDB. + pub fn as_hashdb(&self) -> &HashDB { + self.db.as_hashdb() + } + + /// Returns an interface to mutable HashDB. + pub fn as_hashdb_mut(&mut self) -> &mut HashDB { + self.db.as_hashdb_mut() + } + + /// Clone the database. + pub fn boxed_clone(&self) -> StateDB { + StateDB { + db: self.db.boxed_clone(), + account_cache: self.account_cache.clone(), + cache_overlay: Vec::new(), + is_canon: false, + } + } + + /// Clone the database for a canonical state. + pub fn boxed_clone_canon(&self) -> StateDB { + StateDB { + db: self.db.boxed_clone(), + account_cache: self.account_cache.clone(), + cache_overlay: Vec::new(), + is_canon: true, + } + } + + /// Check if pruning is enabled on the database. + pub fn is_pruned(&self) -> bool { + self.db.is_pruned() + } + + /// Heap size used. + pub fn mem_used(&self) -> usize { + self.db.mem_used() //TODO: + self.account_cache.lock().heap_size_of_children() + } + + /// Returns underlying `JournalDB`. + pub fn journal_db(&self) -> &JournalDB { + &*self.db + } + + /// Enqueue cache change. + pub fn cache_account(&mut self, addr: Address, data: Option) { + self.cache_overlay.push((addr, data)); + } + + /// Apply pending cache changes. + fn commit_cache(&mut self) { + let mut cache = self.account_cache.lock(); + for (address, account) in self.cache_overlay.drain(..) { + if let Some(&mut Some(ref mut existing)) = cache.accounts.get_mut(&address) { + if let Some(new) = account { + existing.merge_with(new); + continue; + } + } + cache.accounts.insert(address, account); + } + } + + /// Clear the cache. + pub fn clear_cache(&mut self) { + self.cache_overlay.clear(); + let mut cache = self.account_cache.lock(); + cache.accounts.clear(); + } + + /// Get basic copy of the cached account. Does not include storage. + /// Returns 'None' if the state is non-canonical and cache is disabled + /// or if the account is not cached. + pub fn get_cached_account(&self, addr: &Address) -> Option> { + if !self.is_canon { + return None; + } + let mut cache = self.account_cache.lock(); + cache.accounts.get_mut(&addr).map(|a| a.as_ref().map(|a| a.clone_basic())) + } + + /// Get value from a cached account. + /// Returns 'None' if the state is non-canonical and cache is disabled + /// or if the account is not cached. + pub fn get_cached(&self, a: &Address, f: F) -> Option + where F: FnOnce(Option<&mut Account>) -> U { + if !self.is_canon { + return None; + } + let mut cache = self.account_cache.lock(); + cache.accounts.get_mut(a).map(|c| f(c.as_mut())) + } +} + diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index c1f99f434..6504ef8a9 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -19,6 +19,7 @@ use io::*; use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; +use state_db::StateDB; use block::{OpenBlock, Drain}; use blockchain::{BlockChain, Config as BlockChainConfig}; use state::*; @@ -146,9 +147,9 @@ pub fn generate_dummy_client_with_spec_and_data(get_test_spec: F, block_numbe ).unwrap(); let test_engine = &*test_spec.engine; - let mut db_result = get_temp_journal_db(); + let mut db_result = get_temp_state_db(); let mut db = db_result.take(); - test_spec.ensure_db_good(db.as_hashdb_mut()).unwrap(); + test_spec.ensure_db_good(&mut db).unwrap(); let genesis_header = test_spec.genesis_header(); let mut rolling_timestamp = 40; @@ -321,9 +322,9 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { } } -pub fn get_temp_journal_db() -> GuardedTempResult> { +pub fn get_temp_state_db() -> GuardedTempResult { let temp = RandomTempPath::new(); - let journal_db = get_temp_journal_db_in(temp.as_path()); + let journal_db = get_temp_state_db_in(temp.as_path()); GuardedTempResult { _temp: temp, @@ -333,7 +334,7 @@ pub fn get_temp_journal_db() -> GuardedTempResult> { pub fn get_temp_state() -> GuardedTempResult { let temp = RandomTempPath::new(); - let journal_db = get_temp_journal_db_in(temp.as_path()); + let journal_db = get_temp_state_db_in(temp.as_path()); GuardedTempResult { _temp: temp, @@ -341,13 +342,14 @@ pub fn get_temp_state() -> GuardedTempResult { } } -pub fn get_temp_journal_db_in(path: &Path) -> Box { +pub fn get_temp_state_db_in(path: &Path) -> StateDB { let db = new_db(path.to_str().expect("Only valid utf8 paths for tests.")); - journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, None) + let journal_db = journaldb::new(db.clone(), journaldb::Algorithm::EarlyMerge, None); + StateDB::new(journal_db) } pub fn get_temp_state_in(path: &Path) -> State { - let journal_db = get_temp_journal_db_in(path); + let journal_db = get_temp_state_db_in(path); State::new(journal_db, U256::from(0), Default::default()) } diff --git a/ethcore/src/trace/executive_tracer.rs b/ethcore/src/trace/executive_tracer.rs index b086b2378..ca9bc30b5 100644 --- a/ethcore/src/trace/executive_tracer.rs +++ b/ethcore/src/trace/executive_tracer.rs @@ -31,7 +31,7 @@ fn top_level_subtraces(traces: &[FlatTrace]) -> usize { traces.iter().filter(|t| t.trace_address.is_empty()).count() } -fn update_trace_address(traces: Vec) -> Vec { +fn prefix_subtrace_addresses(mut traces: Vec) -> Vec { // input traces are expected to be ordered like // [] // [0] @@ -48,26 +48,38 @@ fn update_trace_address(traces: Vec) -> Vec { // [0, 0, 1] // [1] // [1, 0] - let mut top_subtrace_index = 0; - let mut subtrace_subtraces_left = 0; - traces.into_iter().map(|mut trace| { - let is_top_subtrace = trace.trace_address.is_empty(); - let is_subtrace = trace.trace_address.len() == 1; - trace.trace_address.push_front(top_subtrace_index); - - if is_top_subtrace { - subtrace_subtraces_left = trace.subtraces; - } else if is_subtrace { - subtrace_subtraces_left -= 1; - } - - if subtrace_subtraces_left == 0 { - top_subtrace_index += 1; - } - trace - }).collect() + let mut current_subtrace_index = 0; + let mut first = true; + for trace in traces.iter_mut() { + match (first, trace.trace_address.is_empty()) { + (true, _) => first = false, + (_, true) => current_subtrace_index += 1, + _ => {} + } + trace.trace_address.push_front(current_subtrace_index); + } + traces } +#[test] +fn should_prefix_address_properly() { + use super::trace::{Action, Res, Suicide}; + + let f = |v: Vec| FlatTrace { + action: Action::Suicide(Suicide { + address: Default::default(), + balance: Default::default(), + refund_address: Default::default(), + }), + result: Res::None, + subtraces: 0, + trace_address: v.into_iter().collect(), + }; + let t = vec![vec![], vec![0], vec![0, 0], vec![0], vec![], vec![], vec![0], vec![]].into_iter().map(&f).collect(); + let t = prefix_subtrace_addresses(t); + assert_eq!(t, vec![vec![0], vec![0, 0], vec![0, 0, 0], vec![0, 0], vec![1], vec![2], vec![2, 0], vec![3]].into_iter().map(&f).collect::>()); +} + impl Tracer for ExecutiveTracer { fn prepare_trace_call(&self, params: &ActionParams) -> Option { Some(Call::from(params.clone())) @@ -93,7 +105,7 @@ impl Tracer for ExecutiveTracer { }; debug!(target: "trace", "Traced call {:?}", trace); self.traces.push(trace); - self.traces.extend(update_trace_address(subs)); + self.traces.extend(prefix_subtrace_addresses(subs)); } fn trace_create(&mut self, create: Option, gas_used: U256, code: Option, address: Address, subs: Vec) { @@ -109,7 +121,7 @@ impl Tracer for ExecutiveTracer { }; debug!(target: "trace", "Traced create {:?}", trace); self.traces.push(trace); - self.traces.extend(update_trace_address(subs)); + self.traces.extend(prefix_subtrace_addresses(subs)); } fn trace_failed_call(&mut self, call: Option, subs: Vec, error: TraceError) { @@ -121,7 +133,7 @@ impl Tracer for ExecutiveTracer { }; debug!(target: "trace", "Traced failed call {:?}", trace); self.traces.push(trace); - self.traces.extend(update_trace_address(subs)); + self.traces.extend(prefix_subtrace_addresses(subs)); } fn trace_failed_create(&mut self, create: Option, subs: Vec, error: TraceError) { @@ -133,7 +145,7 @@ impl Tracer for ExecutiveTracer { }; debug!(target: "trace", "Traced failed create {:?}", trace); self.traces.push(trace); - self.traces.extend(update_trace_address(subs)); + self.traces.extend(prefix_subtrace_addresses(subs)); } fn trace_suicide(&mut self, address: Address, balance: U256, refund_address: Address) { diff --git a/ethcore/src/types/block_status.rs b/ethcore/src/types/block_status.rs index bf8218e47..857daae10 100644 --- a/ethcore/src/types/block_status.rs +++ b/ethcore/src/types/block_status.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . //! Block status description module +use verification::queue::Status as QueueStatus; /// General block status #[derive(Debug, Eq, PartialEq, Binary)] @@ -28,3 +29,13 @@ pub enum BlockStatus { /// Unknown. Unknown, } + +impl From for BlockStatus { + fn from(status: QueueStatus) -> Self { + match status { + QueueStatus::Queued => BlockStatus::Queued, + QueueStatus::Bad => BlockStatus::Bad, + QueueStatus::Unknown => BlockStatus::Unknown, + } + } +} \ No newline at end of file diff --git a/ethcore/src/types/mod.rs.in b/ethcore/src/types/mod.rs.in index 0537fe056..32c7faabe 100644 --- a/ethcore/src/types/mod.rs.in +++ b/ethcore/src/types/mod.rs.in @@ -25,7 +25,7 @@ pub mod executed; pub mod block_status; pub mod account_diff; pub mod state_diff; -pub mod block_queue_info; +pub mod verification_queue_info; pub mod filter; pub mod trace_filter; pub mod call_analytics; diff --git a/ethcore/src/types/transaction.rs b/ethcore/src/types/transaction.rs index 386b85f7e..f32a2f4dd 100644 --- a/ethcore/src/types/transaction.rs +++ b/ethcore/src/types/transaction.rs @@ -20,7 +20,7 @@ use std::ops::Deref; use std::cell::*; use rlp::*; use util::sha3::Hashable; -use util::{H256, Address, U256, Bytes}; +use util::{H256, Address, U256, Bytes, HeapSizeOf}; use ethkey::{Signature, sign, Secret, Public, recover, public_to_address, Error as EthkeyError}; use error::*; use evm::Schedule; @@ -86,6 +86,12 @@ impl Transaction { } } +impl HeapSizeOf for Transaction { + fn heap_size_of_children(&self) -> usize { + self.data.heap_size_of_children() + } +} + impl From for SignedTransaction { fn from(t: ethjson::state::Transaction) -> Self { let to: Option = t.to.into(); @@ -251,6 +257,12 @@ impl Encodable for SignedTransaction { fn rlp_append(&self, s: &mut RlpStream) { self.rlp_append_sealed_transaction(s) } } +impl HeapSizeOf for SignedTransaction { + fn heap_size_of_children(&self) -> usize { + self.unsigned.heap_size_of_children() + } +} + impl SignedTransaction { /// Append object with a signature into RLP stream pub fn rlp_append_sealed_transaction(&self, s: &mut RlpStream) { diff --git a/ethcore/src/types/verification_queue_info.rs b/ethcore/src/types/verification_queue_info.rs new file mode 100644 index 000000000..35954e7a9 --- /dev/null +++ b/ethcore/src/types/verification_queue_info.rs @@ -0,0 +1,53 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Verification queue info types + +/// Verification queue status +#[derive(Debug, Binary)] +pub struct VerificationQueueInfo { + /// Number of queued items pending verification + pub unverified_queue_size: usize, + /// Number of verified queued items pending import + pub verified_queue_size: usize, + /// Number of items being verified + pub verifying_queue_size: usize, + /// Configured maximum number of items in the queue + pub max_queue_size: usize, + /// Configured maximum number of bytes to use + pub max_mem_use: usize, + /// Heap memory used in bytes + pub mem_used: usize, +} + +impl VerificationQueueInfo { + /// The total size of the queues. + pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size } + + /// The size of the unverified and verifying queues. + pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size } + + /// Indicates that queue is full + pub fn is_full(&self) -> bool { + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || + self.mem_used > self.max_mem_use + } + + /// Indicates that queue is empty + pub fn is_empty(&self) -> bool { + self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size == 0 + } +} \ No newline at end of file diff --git a/ethcore/src/verification/mod.rs b/ethcore/src/verification/mod.rs index ed9c8ebc7..ed9e0ec4c 100644 --- a/ethcore/src/verification/mod.rs +++ b/ethcore/src/verification/mod.rs @@ -16,6 +16,7 @@ pub mod verification; pub mod verifier; +pub mod queue; mod canon_verifier; mod noop_verifier; @@ -23,6 +24,7 @@ pub use self::verification::*; pub use self::verifier::Verifier; pub use self::canon_verifier::CanonVerifier; pub use self::noop_verifier::NoopVerifier; +pub use self::queue::{BlockQueue, Config as QueueConfig, VerificationQueue, QueueInfo}; /// Verifier type. #[derive(Debug, PartialEq, Clone)] diff --git a/ethcore/src/verification/queue/kind.rs b/ethcore/src/verification/queue/kind.rs new file mode 100644 index 000000000..7585f1e6d --- /dev/null +++ b/ethcore/src/verification/queue/kind.rs @@ -0,0 +1,182 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Definition of valid items for the verification queue. + +use engines::Engine; +use error::Error; + +use util::{HeapSizeOf, H256}; + +pub use self::blocks::Blocks; +pub use self::headers::Headers; + +/// Something which can produce a hash and a parent hash. +pub trait HasHash { + /// Get the hash of this item. + fn hash(&self) -> H256; + + /// Get the hash of this item's parent. + fn parent_hash(&self) -> H256; +} + +/// Defines transitions between stages of verification. +/// +/// It starts with a fallible transformation from an "input" into the unverified item. +/// This consists of quick, simply done checks as well as extracting particular data. +/// +/// Then, there is a `verify` function which performs more expensive checks and +/// produces the verified output. +/// +/// For correctness, the hashes produced by each stage of the pipeline should be +/// consistent. +pub trait Kind: 'static + Sized + Send + Sync { + /// The first stage: completely unverified. + type Input: Sized + Send + HasHash + HeapSizeOf; + + /// The second stage: partially verified. + type Unverified: Sized + Send + HasHash + HeapSizeOf; + + /// The third stage: completely verified. + type Verified: Sized + Send + HasHash + HeapSizeOf; + + /// Attempt to create the `Unverified` item from the input. + fn create(input: Self::Input, engine: &Engine) -> Result; + + /// Attempt to verify the `Unverified` item using the given engine. + fn verify(unverified: Self::Unverified, engine: &Engine) -> Result; +} + +/// The blocks verification module. +pub mod blocks { + use super::{Kind, HasHash}; + + use engines::Engine; + use error::Error; + use header::Header; + use verification::{PreverifiedBlock, verify_block_basic, verify_block_unordered}; + + use util::{Bytes, HeapSizeOf, H256}; + + /// A mode for verifying blocks. + pub struct Blocks; + + impl Kind for Blocks { + type Input = Unverified; + type Unverified = Unverified; + type Verified = PreverifiedBlock; + + fn create(input: Self::Input, engine: &Engine) -> Result { + match verify_block_basic(&input.header, &input.bytes, engine) { + Ok(()) => Ok(input), + Err(e) => { + warn!(target: "client", "Stage 1 block verification failed for {}: {:?}", input.hash(), e); + Err(e) + } + } + } + + fn verify(un: Self::Unverified, engine: &Engine) -> Result { + let hash = un.hash(); + match verify_block_unordered(un.header, un.bytes, engine) { + Ok(verified) => Ok(verified), + Err(e) => { + warn!(target: "client", "Stage 2 block verification failed for {}: {:?}", hash, e); + Err(e) + } + } + } + } + + /// An unverified block. + pub struct Unverified { + header: Header, + bytes: Bytes, + } + + impl Unverified { + /// Create an `Unverified` from raw bytes. + pub fn new(bytes: Bytes) -> Self { + use views::BlockView; + + let header = BlockView::new(&bytes).header(); + Unverified { + header: header, + bytes: bytes, + } + } + } + + impl HeapSizeOf for Unverified { + fn heap_size_of_children(&self) -> usize { + self.header.heap_size_of_children() + self.bytes.heap_size_of_children() + } + } + + impl HasHash for Unverified { + fn hash(&self) -> H256 { + self.header.hash() + } + + fn parent_hash(&self) -> H256 { + self.header.parent_hash().clone() + } + } + + impl HasHash for PreverifiedBlock { + fn hash(&self) -> H256 { + self.header.hash() + } + + fn parent_hash(&self) -> H256 { + self.header.parent_hash().clone() + } + } +} + +/// Verification for headers. +pub mod headers { + use super::{Kind, HasHash}; + + use engines::Engine; + use error::Error; + use header::Header; + use verification::verify_header_params; + + use util::hash::H256; + + impl HasHash for Header { + fn hash(&self) -> H256 { self.hash() } + fn parent_hash(&self) -> H256 { self.parent_hash().clone() } + } + + /// A mode for verifying headers. + pub struct Headers; + + impl Kind for Headers { + type Input = Header; + type Unverified = Header; + type Verified = Header; + + fn create(input: Self::Input, engine: &Engine) -> Result { + verify_header_params(&input, engine).map(|_| input) + } + + fn verify(unverified: Self::Unverified, engine: &Engine) -> Result { + engine.verify_block_unordered(&unverified, None).map(|_| unverified) + } + } +} \ No newline at end of file diff --git a/ethcore/src/block_queue.rs b/ethcore/src/verification/queue/mod.rs similarity index 65% rename from ethcore/src/block_queue.rs rename to ethcore/src/verification/queue/mod.rs index c441136fd..3f81d53ce 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/verification/queue/mod.rs @@ -16,30 +16,35 @@ //! A queue of blocks. Sits between network or other I/O and the `BlockChain`. //! Sorts them ready for blockchain insertion. + use std::thread::{JoinHandle, self}; use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; use std::sync::{Condvar as SCondvar, Mutex as SMutex}; use util::*; use io::*; -use verification::*; use error::*; use engines::Engine; -use views::*; -use header::*; use service::*; -use client::BlockStatus; -pub use types::block_queue_info::BlockQueueInfo; +use self::kind::{HasHash, Kind}; -known_heap_size!(0, UnverifiedBlock, VerifyingBlock, PreverifiedBlock); +pub use types::verification_queue_info::VerificationQueueInfo as QueueInfo; + +pub mod kind; const MIN_MEM_LIMIT: usize = 16384; const MIN_QUEUE_LIMIT: usize = 512; -/// Block queue configuration +/// Type alias for block queue convenience. +pub type BlockQueue = VerificationQueue; + +/// Type alias for header queue convenience. +pub type HeaderQueue = VerificationQueue; + +/// Verification queue configuration #[derive(Debug, PartialEq, Clone)] -pub struct BlockQueueConfig { - /// Maximum number of blocks to keep in unverified queue. +pub struct Config { + /// Maximum number of items to keep in unverified queue. /// When the limit is reached, is_full returns true. pub max_queue_size: usize, /// Maximum heap memory to use. @@ -47,42 +52,44 @@ pub struct BlockQueueConfig { pub max_mem_use: usize, } -impl Default for BlockQueueConfig { +impl Default for Config { fn default() -> Self { - BlockQueueConfig { + Config { max_queue_size: 30000, max_mem_use: 50 * 1024 * 1024, } } } +/// An item which is in the process of being verified. +pub struct Verifying { + hash: H256, + output: Option, +} -impl BlockQueueInfo { - /// The total size of the queues. - pub fn total_queue_size(&self) -> usize { self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size } - - /// The size of the unverified and verifying queues. - pub fn incomplete_queue_size(&self) -> usize { self.unverified_queue_size + self.verifying_queue_size } - - /// Indicates that queue is full - pub fn is_full(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size > self.max_queue_size || - self.mem_used > self.max_mem_use - } - - /// Indicates that queue is empty - pub fn is_empty(&self) -> bool { - self.unverified_queue_size + self.verified_queue_size + self.verifying_queue_size == 0 +impl HeapSizeOf for Verifying { + fn heap_size_of_children(&self) -> usize { + self.output.heap_size_of_children() } } -/// A queue of blocks. Sits between network or other I/O and the `BlockChain`. -/// Sorts them ready for blockchain insertion. -pub struct BlockQueue { +/// Status of items in the queue. +pub enum Status { + /// Currently queued. + Queued, + /// Known to be bad. + Bad, + /// Unknown. + Unknown, +} + +/// A queue of items to be verified. Sits between network or other I/O and the `BlockChain`. +/// Keeps them in the same order as inserted, minus invalid items. +pub struct VerificationQueue { panic_handler: Arc, engine: Arc, more_to_verify: Arc, - verification: Arc, + verification: Arc>, verifiers: Vec>, deleting: Arc, ready_signal: Arc, @@ -92,16 +99,6 @@ pub struct BlockQueue { max_mem_use: usize, } -struct UnverifiedBlock { - header: Header, - bytes: Bytes, -} - -struct VerifyingBlock { - hash: H256, - block: Option, -} - struct QueueSignal { deleting: Arc, signalled: AtomicBool, @@ -128,19 +125,19 @@ impl QueueSignal { } } -struct Verification { +struct Verification { // All locks must be captured in the order declared here. - unverified: Mutex>, - verified: Mutex>, - verifying: Mutex>, + unverified: Mutex>, + verified: Mutex>, + verifying: Mutex>>, bad: Mutex>, more_to_verify: SMutex<()>, empty: SMutex<()>, } -impl BlockQueue { +impl VerificationQueue { /// Creates a new queue instance. - pub fn new(config: BlockQueueConfig, engine: Arc, message_channel: IoChannel) -> BlockQueue { + pub fn new(config: Config, engine: Arc, message_channel: IoChannel) -> Self { let verification = Arc::new(Verification { unverified: Mutex::new(VecDeque::new()), verified: Mutex::new(VecDeque::new()), @@ -175,13 +172,13 @@ impl BlockQueue { .name(format!("Verifier #{}", i)) .spawn(move || { panic_handler.catch_panic(move || { - BlockQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) + VerificationQueue::verify(verification, engine, more_to_verify, ready_signal, deleting, empty) }).unwrap() }) .expect("Error starting block verification thread") ); } - BlockQueue { + VerificationQueue { engine: engine, panic_handler: panic_handler, ready_signal: ready_signal.clone(), @@ -196,7 +193,7 @@ impl BlockQueue { } } - fn verify(verification: Arc, engine: Arc, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { + fn verify(verification: Arc>, engine: Arc, wait: Arc, ready: Arc, deleting: Arc, empty: Arc) { while !deleting.load(AtomicOrdering::Acquire) { { let mut more_to_verify = verification.more_to_verify.lock().unwrap(); @@ -214,57 +211,66 @@ impl BlockQueue { } } - let block = { + let item = { + // acquire these locks before getting the item to verify. let mut unverified = verification.unverified.lock(); - if unverified.is_empty() { - continue; - } let mut verifying = verification.verifying.lock(); - let block = unverified.pop_front().unwrap(); - verifying.push_back(VerifyingBlock{ hash: block.header.hash(), block: None }); - block + + let item = match unverified.pop_front() { + Some(item) => item, + None => continue, + }; + + verifying.push_back(Verifying { hash: item.hash(), output: None }); + item }; - let block_hash = block.header.hash(); - match verify_block_unordered(block.header, block.bytes, &*engine) { + let hash = item.hash(); + match K::verify(item, &*engine) { Ok(verified) => { let mut verifying = verification.verifying.lock(); - for e in verifying.iter_mut() { - if e.hash == block_hash { - e.block = Some(verified); + let mut idx = None; + for (i, e) in verifying.iter_mut().enumerate() { + if e.hash == hash { + idx = Some(i); + e.output = Some(verified); break; } } - if !verifying.is_empty() && verifying.front().unwrap().hash == block_hash { + + if idx == Some(0) { // we're next! let mut verified = verification.verified.lock(); let mut bad = verification.bad.lock(); - BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); + VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); ready.set(); } }, - Err(err) => { + Err(_) => { let mut verifying = verification.verifying.lock(); let mut verified = verification.verified.lock(); let mut bad = verification.bad.lock(); - warn!(target: "client", "Stage 2 block verification failed for {}\nError: {:?}", block_hash, err); - bad.insert(block_hash.clone()); - verifying.retain(|e| e.hash != block_hash); - BlockQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); - ready.set(); + + bad.insert(hash.clone()); + verifying.retain(|e| e.hash != hash); + + if verifying.front().map_or(false, |x| x.output.is_some()) { + VerificationQueue::drain_verifying(&mut verifying, &mut verified, &mut bad); + ready.set(); + } } } } } - fn drain_verifying(verifying: &mut VecDeque, verified: &mut VecDeque, bad: &mut HashSet) { - while !verifying.is_empty() && verifying.front().unwrap().block.is_some() { - let block = verifying.pop_front().unwrap().block.unwrap(); - if bad.contains(block.header.parent_hash()) { - bad.insert(block.header.hash()); - } - else { - verified.push_back(block); + fn drain_verifying(verifying: &mut VecDeque>, verified: &mut VecDeque, bad: &mut HashSet) { + while let Some(output) = verifying.front_mut().and_then(|x| x.output.take()) { + assert!(verifying.pop_front().is_some()); + + if bad.contains(&output.parent_hash()) { + bad.insert(output.hash()); + } else { + verified.push_back(output); } } } @@ -288,21 +294,20 @@ impl BlockQueue { } } - /// Check if the block is currently in the queue - pub fn block_status(&self, hash: &H256) -> BlockStatus { + /// Check if the item is currently in the queue + pub fn status(&self, hash: &H256) -> Status { if self.processing.read().contains(hash) { - return BlockStatus::Queued; + return Status::Queued; } if self.verification.bad.lock().contains(hash) { - return BlockStatus::Bad; + return Status::Bad; } - BlockStatus::Unknown + Status::Unknown } /// Add a block to the queue. - pub fn import_block(&self, bytes: Bytes) -> ImportResult { - let header = BlockView::new(&bytes).header(); - let h = header.hash(); + pub fn import(&self, input: K::Input) -> ImportResult { + let h = input.hash(); { if self.processing.read().contains(&h) { return Err(ImportError::AlreadyQueued.into()); @@ -313,74 +318,71 @@ impl BlockQueue { return Err(ImportError::KnownBad.into()); } - if bad.contains(header.parent_hash()) { + if bad.contains(&input.parent_hash()) { bad.insert(h.clone()); return Err(ImportError::KnownBad.into()); } } - match verify_block_basic(&header, &bytes, &*self.engine) { - Ok(()) => { + match K::create(input, &*self.engine) { + Ok(item) => { self.processing.write().insert(h.clone()); - self.verification.unverified.lock().push_back(UnverifiedBlock { header: header, bytes: bytes }); + self.verification.unverified.lock().push_back(item); self.more_to_verify.notify_all(); Ok(h) }, Err(err) => { - warn!(target: "client", "Stage 1 block verification failed for {}\nError: {:?}", BlockView::new(&bytes).header_view().sha3(), err); self.verification.bad.lock().insert(h.clone()); Err(err) } } } - /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&self, block_hashes: &[H256]) { - if block_hashes.is_empty() { + /// Mark given item and all its children as bad. pauses verification + /// until complete. + pub fn mark_as_bad(&self, hashes: &[H256]) { + if hashes.is_empty() { return; } let mut verified_lock = self.verification.verified.lock(); let mut verified = &mut *verified_lock; let mut bad = self.verification.bad.lock(); let mut processing = self.processing.write(); - bad.reserve(block_hashes.len()); - for hash in block_hashes { + bad.reserve(hashes.len()); + for hash in hashes { bad.insert(hash.clone()); processing.remove(hash); } let mut new_verified = VecDeque::new(); - for block in verified.drain(..) { - if bad.contains(block.header.parent_hash()) { - bad.insert(block.header.hash()); - processing.remove(&block.header.hash()); + for output in verified.drain(..) { + if bad.contains(&output.parent_hash()) { + bad.insert(output.hash()); + processing.remove(&output.hash()); } else { - new_verified.push_back(block); + new_verified.push_back(output); } } *verified = new_verified; } - /// Mark given block as processed - pub fn mark_as_good(&self, block_hashes: &[H256]) { - if block_hashes.is_empty() { + /// Mark given item as processed + pub fn mark_as_good(&self, hashes: &[H256]) { + if hashes.is_empty() { return; } let mut processing = self.processing.write(); - for hash in block_hashes { + for hash in hashes { processing.remove(hash); } } - /// Removes up to `max` verified blocks from the queue - pub fn drain(&self, max: usize) -> Vec { + /// Removes up to `max` verified items from the queue + pub fn drain(&self, max: usize) -> Vec { let mut verified = self.verification.verified.lock(); let count = min(max, verified.len()); - let mut result = Vec::with_capacity(count); - for _ in 0..count { - let block = verified.pop_front().unwrap(); - result.push(block); - } + let result = verified.drain(..count).collect::>(); + self.ready_signal.reset(); if !verified.is_empty() { self.ready_signal.set(); @@ -389,7 +391,7 @@ impl BlockQueue { } /// Get queue status. - pub fn queue_info(&self) -> BlockQueueInfo { + pub fn queue_info(&self) -> QueueInfo { let (unverified_len, unverified_bytes) = { let v = self.verification.unverified.lock(); (v.len(), v.heap_size_of_children()) @@ -402,7 +404,8 @@ impl BlockQueue { let v = self.verification.verified.lock(); (v.len(), v.heap_size_of_children()) }; - BlockQueueInfo { + + QueueInfo { unverified_queue_size: unverified_len, verifying_queue_size: verifying_len, verified_queue_size: verified_len, @@ -428,22 +431,22 @@ impl BlockQueue { } } -impl MayPanic for BlockQueue { +impl MayPanic for VerificationQueue { fn on_panic(&self, closure: F) where F: OnPanicListener { self.panic_handler.on_panic(closure); } } -impl Drop for BlockQueue { +impl Drop for VerificationQueue { fn drop(&mut self) { - trace!(target: "shutdown", "[BlockQueue] Closing..."); + trace!(target: "shutdown", "[VerificationQueue] Closing..."); self.clear(); self.deleting.store(true, AtomicOrdering::Release); self.more_to_verify.notify_all(); for t in self.verifiers.drain(..) { t.join().unwrap(); } - trace!(target: "shutdown", "[BlockQueue] Closed."); + trace!(target: "shutdown", "[VerificationQueue] Closed."); } } @@ -452,7 +455,8 @@ mod tests { use util::*; use io::*; use spec::*; - use block_queue::*; + use super::{BlockQueue, Config}; + use super::kind::blocks::Unverified; use tests::helpers::*; use error::*; use views::*; @@ -460,7 +464,7 @@ mod tests { fn get_test_queue() -> BlockQueue { let spec = get_test_spec(); let engine = spec.engine; - BlockQueue::new(BlockQueueConfig::default(), engine, IoChannel::disconnected()) + BlockQueue::new(Config::default(), engine, IoChannel::disconnected()) } #[test] @@ -468,13 +472,13 @@ mod tests { // TODO better test let spec = Spec::new_test(); let engine = spec.engine; - let _ = BlockQueue::new(BlockQueueConfig::default(), engine, IoChannel::disconnected()); + let _ = BlockQueue::new(Config::default(), engine, IoChannel::disconnected()); } #[test] fn can_import_blocks() { let queue = get_test_queue(); - if let Err(e) = queue.import_block(get_good_dummy_block()) { + if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) { panic!("error importing block that is valid by definition({:?})", e); } } @@ -482,11 +486,11 @@ mod tests { #[test] fn returns_error_for_duplicates() { let queue = get_test_queue(); - if let Err(e) = queue.import_block(get_good_dummy_block()) { + if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) { panic!("error importing block that is valid by definition({:?})", e); } - let duplicate_import = queue.import_block(get_good_dummy_block()); + let duplicate_import = queue.import(Unverified::new(get_good_dummy_block())); match duplicate_import { Err(e) => { match e { @@ -503,14 +507,14 @@ mod tests { let queue = get_test_queue(); let block = get_good_dummy_block(); let hash = BlockView::new(&block).header().hash().clone(); - if let Err(e) = queue.import_block(block) { + if let Err(e) = queue.import(Unverified::new(block)) { panic!("error importing block that is valid by definition({:?})", e); } queue.flush(); queue.drain(10); queue.mark_as_good(&[ hash ]); - if let Err(e) = queue.import_block(get_good_dummy_block()) { + if let Err(e) = queue.import(Unverified::new(get_good_dummy_block())) { panic!("error importing block that has already been drained ({:?})", e); } } @@ -518,7 +522,8 @@ mod tests { #[test] fn returns_empty_once_finished() { let queue = get_test_queue(); - queue.import_block(get_good_dummy_block()).expect("error importing block that is valid by definition"); + queue.import(Unverified::new(get_good_dummy_block())) + .expect("error importing block that is valid by definition"); queue.flush(); queue.drain(1); @@ -529,13 +534,13 @@ mod tests { fn test_mem_limit() { let spec = get_test_spec(); let engine = spec.engine; - let mut config = BlockQueueConfig::default(); + let mut config = Config::default(); config.max_mem_use = super::MIN_MEM_LIMIT; // empty queue uses about 15000 let queue = BlockQueue::new(config, engine, IoChannel::disconnected()); assert!(!queue.queue_info().is_full()); let mut blocks = get_good_dummy_block_seq(50); for b in blocks.drain(..) { - queue.import_block(b).unwrap(); + queue.import(Unverified::new(b)).unwrap(); } assert!(queue.queue_info().is_full()); } diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 4e1305a33..f89ac7d9a 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -36,14 +36,22 @@ pub struct PreverifiedBlock { pub bytes: Bytes, } +impl HeapSizeOf for PreverifiedBlock { + fn heap_size_of_children(&self) -> usize { + self.header.heap_size_of_children() + + self.transactions.heap_size_of_children() + + self.bytes.heap_size_of_children() + } +} + /// Phase 1 quick block verification. Only does checks that are cheap. Operates on a single block pub fn verify_block_basic(header: &Header, bytes: &[u8], engine: &Engine) -> Result<(), Error> { - try!(verify_header(&header, engine)); + try!(verify_header_params(&header, engine)); try!(verify_block_integrity(bytes, &header.transactions_root(), &header.uncles_hash())); try!(engine.verify_block_basic(&header, Some(bytes))); for u in try!(UntrustedRlp::new(bytes).at(2)).iter().map(|rlp| rlp.as_val::
()) { let u = try!(u); - try!(verify_header(&u, engine)); + try!(verify_header_params(&u, engine)); try!(engine.verify_block_basic(&u, None)); } // Verify transactions. @@ -179,7 +187,7 @@ pub fn verify_block_final(expected: &Header, got: &Header) -> Result<(), Error> } /// Check basic header parameters. -fn verify_header(header: &Header, engine: &Engine) -> Result<(), Error> { +pub fn verify_header_params(header: &Header, engine: &Engine) -> Result<(), Error> { if header.number() >= From::from(BlockNumber::max_value()) { return Err(From::from(BlockError::RidiculousNumber(OutOfBounds { max: Some(From::from(BlockNumber::max_value())), min: None, found: header.number() }))) } diff --git a/ethkey/src/signature.rs b/ethkey/src/signature.rs index 8733f1245..1bad5b83e 100644 --- a/ethkey/src/signature.rs +++ b/ethkey/src/signature.rs @@ -26,7 +26,6 @@ use bigint::hash::{H520, H256, FixedHash}; use {Secret, Public, SECP256K1, Error, Message, public_to_address, Address}; #[repr(C)] -#[derive(Eq)] pub struct Signature([u8; 65]); impl Signature { @@ -77,6 +76,9 @@ impl PartialEq for Signature { } } +// manual implementation required in Rust 1.13+, see `std::cmp::AssertParamIsEq`. +impl Eq for Signature { } + // also manual for the same reason, but the pretty printing might be useful. impl fmt::Debug for Signature { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { diff --git a/ethstore/src/account/safe_account.rs b/ethstore/src/account/safe_account.rs index 38069a718..315fd283b 100644 --- a/ethstore/src/account/safe_account.rs +++ b/ethstore/src/account/safe_account.rs @@ -16,14 +16,14 @@ use ethkey::{KeyPair, sign, Address, Secret, Signature, Message}; use {json, Error, crypto}; -use crypto::{Keccak256}; +use crypto::Keccak256; use random::Random; use account::{Version, Cipher, Kdf, Aes128Ctr, Pbkdf2, Prf}; #[derive(Debug, PartialEq, Clone)] pub struct Crypto { pub cipher: Cipher, - pub ciphertext: [u8; 32], + pub ciphertext: Vec, pub kdf: Kdf, pub mac: [u8; 32], } @@ -95,7 +95,7 @@ impl Crypto { cipher: Cipher::Aes128Ctr(Aes128Ctr { iv: iv, }), - ciphertext: ciphertext, + ciphertext: ciphertext.to_vec(), kdf: Kdf::Pbkdf2(Pbkdf2 { dklen: crypto::KEY_LENGTH as u32, salt: salt, @@ -107,6 +107,10 @@ impl Crypto { } pub fn secret(&self, password: &str) -> Result { + if self.ciphertext.len() > 32 { + return Err(Error::InvalidSecret); + } + let (derived_left_bits, derived_right_bits) = match self.kdf { Kdf::Pbkdf2(ref params) => crypto::derive_key_iterations(password, ¶ms.salt, params.c), Kdf::Scrypt(ref params) => crypto::derive_key_scrypt(password, ¶ms.salt, params.n, params.p, params.r), @@ -122,7 +126,8 @@ impl Crypto { match self.cipher { Cipher::Aes128Ctr(ref params) => { - crypto::aes::decrypt(&derived_left_bits, ¶ms.iv, &self.ciphertext, &mut *secret) + let from = 32 - self.ciphertext.len(); + crypto::aes::decrypt(&derived_left_bits, ¶ms.iv, &self.ciphertext, &mut (&mut *secret)[from..]) }, } diff --git a/ethstore/src/json/bytes.rs b/ethstore/src/json/bytes.rs new file mode 100644 index 000000000..fd4a3b995 --- /dev/null +++ b/ethstore/src/json/bytes.rs @@ -0,0 +1,58 @@ +use std::{ops, str}; +use serde::{Deserialize, Deserializer, Error, Serialize, Serializer}; +use rustc_serialize::hex::{ToHex, FromHex, FromHexError}; + +#[derive(Debug, PartialEq)] +pub struct Bytes(Vec); + +impl ops::Deref for Bytes { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Deserialize for Bytes { + fn deserialize(deserializer: &mut D) -> Result + where D: Deserializer + { + let s = try!(String::deserialize(deserializer)); + let data = try!(s.from_hex().map_err(|e| Error::custom(format!("Invalid hex value {}", e)))); + Ok(Bytes(data)) + } +} + +impl Serialize for Bytes { + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> + where S: Serializer { + serializer.serialize_str(&self.0.to_hex()) + } +} + +impl str::FromStr for Bytes { + type Err = FromHexError; + + fn from_str(s: &str) -> Result { + s.from_hex().map(Bytes) + } +} + +impl From<&'static str> for Bytes { + fn from(s: &'static str) -> Self { + s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s)) + } +} + +impl From> for Bytes { + fn from(v: Vec) -> Self { + Bytes(v) + } +} + +impl From for Vec { + fn from(b: Bytes) -> Self { + b.0 + } +} + diff --git a/ethstore/src/json/crypto.rs b/ethstore/src/json/crypto.rs index e6ecef81f..739a2fea9 100644 --- a/ethstore/src/json/crypto.rs +++ b/ethstore/src/json/crypto.rs @@ -16,12 +16,14 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer, Error}; use serde::de::{Visitor, MapVisitor}; -use super::{Cipher, CipherSer, CipherSerParams, Kdf, KdfSer, KdfSerParams, H256}; +use super::{Cipher, CipherSer, CipherSerParams, Kdf, KdfSer, KdfSerParams, H256, Bytes}; + +pub type CipherText = Bytes; #[derive(Debug, PartialEq)] pub struct Crypto { pub cipher: Cipher, - pub ciphertext: H256, + pub ciphertext: CipherText, pub kdf: Kdf, pub mac: H256, } diff --git a/ethstore/src/json/hash.rs b/ethstore/src/json/hash.rs index 0079b4f81..25bf51130 100644 --- a/ethstore/src/json/hash.rs +++ b/ethstore/src/json/hash.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::fmt; -use std::ops; -use std::str::FromStr; +use std::{ops, fmt, str}; use rustc_serialize::hex::{FromHex, ToHex}; use serde::{Serialize, Serializer, Deserialize, Deserializer, Error as SerdeError}; use serde::de::Visitor; @@ -65,7 +63,7 @@ macro_rules! impl_hash { type Value = $name; fn visit_str(&mut self, value: &str) -> Result where E: SerdeError { - FromStr::from_str(value).map_err(SerdeError::custom) + value.parse().map_err(SerdeError::custom) } fn visit_string(&mut self, value: String) -> Result where E: SerdeError { @@ -77,7 +75,7 @@ macro_rules! impl_hash { } } - impl FromStr for $name { + impl str::FromStr for $name { type Err = Error; fn from_str(value: &str) -> Result { @@ -92,6 +90,12 @@ macro_rules! impl_hash { } } + impl From<&'static str> for $name { + fn from(s: &'static str) -> Self { + s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s)) + } + } + impl From<[u8; $size]> for $name { fn from(bytes: [u8; $size]) -> Self { $name(bytes) diff --git a/ethstore/src/json/id.rs b/ethstore/src/json/id.rs index 2e896458c..ff282a9f8 100644 --- a/ethstore/src/json/id.rs +++ b/ethstore/src/json/id.rs @@ -15,8 +15,7 @@ // along with Parity. If not, see . //! Universaly unique identifier. -use std::str::FromStr; -use std::fmt; +use std::{fmt, str}; use rustc_serialize::hex::{ToHex, FromHex}; use serde::{Deserialize, Serialize, Deserializer, Serializer, Error as SerdeError}; use serde::de::Visitor; @@ -73,7 +72,7 @@ fn copy_into(from: &str, into: &mut [u8]) -> Result<(), Error> { Ok(()) } -impl FromStr for UUID { +impl str::FromStr for UUID { type Err = Error; fn from_str(s: &str) -> Result { @@ -95,6 +94,12 @@ impl FromStr for UUID { } } +impl From<&'static str> for UUID { + fn from(s: &'static str) -> Self { + s.parse().expect(&format!("invalid string literal for {}: '{}'", stringify!(Self), s)) + } +} + impl Serialize for UUID { fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: Serializer { @@ -116,7 +121,7 @@ impl Visitor for UUIDVisitor { type Value = UUID; fn visit_str(&mut self, value: &str) -> Result where E: SerdeError { - UUID::from_str(value).map_err(SerdeError::custom) + value.parse().map_err(SerdeError::custom) } fn visit_string(&mut self, value: String) -> Result where E: SerdeError { @@ -126,19 +131,18 @@ impl Visitor for UUIDVisitor { #[cfg(test)] mod tests { - use std::str::FromStr; use super::UUID; #[test] fn uuid_from_str() { - let uuid = UUID::from_str("3198bc9c-6672-5ab3-d995-4942343ae5b6").unwrap(); + let uuid: UUID = "3198bc9c-6672-5ab3-d995-4942343ae5b6".into(); assert_eq!(uuid, UUID::from([0x31, 0x98, 0xbc, 0x9c, 0x66, 0x72, 0x5a, 0xb3, 0xd9, 0x95, 0x49, 0x42, 0x34, 0x3a, 0xe5, 0xb6])); } #[test] fn uuid_from_and_to_str() { let from = "3198bc9c-6672-5ab3-d995-4942343ae5b6"; - let uuid = UUID::from_str(from).unwrap(); + let uuid: UUID = from.into(); let to: String = uuid.into(); assert_eq!(from, &to); } diff --git a/ethstore/src/json/key_file.rs b/ethstore/src/json/key_file.rs index 7d970a15c..6e37c7c89 100644 --- a/ethstore/src/json/key_file.rs +++ b/ethstore/src/json/key_file.rs @@ -98,7 +98,7 @@ impl Visitor for KeyFileVisitor { Some(KeyFileField::Version) => { version = Some(try!(visitor.visit_value())); } Some(KeyFileField::Crypto) => { crypto = Some(try!(visitor.visit_value())); } Some(KeyFileField::Address) => { address = Some(try!(visitor.visit_value())); } - Some(KeyFileField::Name) => { name = visitor.visit_value().ok(); } // ignore anyhing that is not a string to be permissive. + Some(KeyFileField::Name) => { name = visitor.visit_value().ok(); } // ignore anyhing that is not a string to be permissive. Some(KeyFileField::Meta) => { meta = visitor.visit_value().ok(); } // ignore anyhing that is not a string to be permissive. None => { break; } } @@ -153,7 +153,7 @@ impl KeyFile { mod tests { use std::str::FromStr; use serde_json; - use json::{KeyFile, UUID, Version, Crypto, Cipher, Aes128Ctr, Kdf, Scrypt, H128, H160, H256}; + use json::{KeyFile, UUID, Version, Crypto, Cipher, Aes128Ctr, Kdf, Scrypt}; #[test] fn basic_keyfile() { @@ -185,20 +185,20 @@ mod tests { let expected = KeyFile { id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(), version: Version::V3, - address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(), + address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(), crypto: Crypto { cipher: Cipher::Aes128Ctr(Aes128Ctr { - iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(), + iv: "b5a7ec855ec9e2c405371356855fec83".into(), }), - ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(), + ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(), kdf: Kdf::Scrypt(Scrypt { n: 262144, dklen: 32, p: 1, r: 8, - salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(), + salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(), }), - mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(), + mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(), }, name: Some("Test".to_owned()), meta: Some("{}".to_owned()), @@ -234,22 +234,22 @@ mod tests { }"#; let expected = KeyFile { - id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(), + id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(), version: Version::V3, - address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(), + address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(), crypto: Crypto { cipher: Cipher::Aes128Ctr(Aes128Ctr { - iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(), + iv: "b5a7ec855ec9e2c405371356855fec83".into(), }), - ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(), + ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(), kdf: Kdf::Scrypt(Scrypt { n: 262144, dklen: 32, p: 1, r: 8, - salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(), + salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(), }), - mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(), + mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(), }, name: None, meta: None, @@ -262,22 +262,22 @@ mod tests { #[test] fn to_and_from_json() { let file = KeyFile { - id: UUID::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(), + id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(), version: Version::V3, - address: H160::from_str("6edddfc6349aff20bc6467ccf276c5b52487f7a8").unwrap(), + address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(), crypto: Crypto { cipher: Cipher::Aes128Ctr(Aes128Ctr { - iv: H128::from_str("b5a7ec855ec9e2c405371356855fec83").unwrap(), + iv: "b5a7ec855ec9e2c405371356855fec83".into(), }), - ciphertext: H256::from_str("7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc").unwrap(), + ciphertext: "7203da0676d141b138cd7f8e1a4365f59cc1aa6978dc5443f364ca943d7cb4bc".into(), kdf: Kdf::Scrypt(Scrypt { n: 262144, dklen: 32, p: 1, r: 8, - salt: H256::from_str("1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209").unwrap(), + salt: "1e8642fdf1f87172492c1412fc62f8db75d796cdfa9c53c3f2b11e44a2a1b209".into(), }), - mac: H256::from_str("46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f").unwrap(), + mac: "46325c5d4e8c991ad2683d525c7854da387138b6ca45068985aa4959fa2b8c8f".into(), }, name: Some("Test".to_owned()), meta: None, diff --git a/ethstore/src/json/mod.rs.in b/ethstore/src/json/mod.rs.in index 4f9fdbfe3..133d9821e 100644 --- a/ethstore/src/json/mod.rs.in +++ b/ethstore/src/json/mod.rs.in @@ -1,3 +1,4 @@ +mod bytes; mod cipher; mod crypto; mod error; @@ -8,8 +9,9 @@ mod key_file; mod presale; mod version; +pub use self::bytes::Bytes; pub use self::cipher::{Cipher, CipherSer, CipherSerParams, Aes128Ctr}; -pub use self::crypto::Crypto; +pub use self::crypto::{Crypto, CipherText}; pub use self::error::Error; pub use self::hash::{H128, H160, H256}; pub use self::id::UUID; diff --git a/ethstore/src/json/presale.rs b/ethstore/src/json/presale.rs index 77394fcb1..d1cffcb6a 100644 --- a/ethstore/src/json/presale.rs +++ b/ethstore/src/json/presale.rs @@ -1,30 +1,8 @@ use std::io::Read; -use std::ops::Deref; use serde_json; -use serde::{Deserialize, Deserializer, Error}; -use rustc_serialize::hex::FromHex; -use super::{H160}; +use super::{H160, Bytes}; -#[derive(Debug, PartialEq)] -pub struct Encseed(Vec); - -impl Deref for Encseed { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Deserialize for Encseed { - fn deserialize(deserializer: &mut D) -> Result - where D: Deserializer - { - let s = try!(String::deserialize(deserializer)); - let data = try!(s.from_hex().map_err(|e| Error::custom(format!("Invalid hex value {}", e)))); - Ok(Encseed(data)) - } -} +pub type Encseed = Bytes; #[derive(Debug, PartialEq, Deserialize)] pub struct PresaleWallet { @@ -43,8 +21,7 @@ impl PresaleWallet { mod tests { use std::str::FromStr; use serde_json; - use rustc_serialize::hex::FromHex; - use json::{PresaleWallet, H160, Encseed}; + use json::{PresaleWallet, H160}; #[test] fn presale_wallet() { @@ -57,7 +34,7 @@ mod tests { } "#; let expected = PresaleWallet { - encseed: Encseed("137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066".from_hex().unwrap()), + encseed: "137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066".into(), address: H160::from_str("ede84640d1a1d3e06902048e67aa7db8d52c2ce1").unwrap(), }; @@ -77,7 +54,7 @@ mod tests { } "#; let expected = PresaleWallet { - encseed: Encseed("137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0d".from_hex().unwrap()), + encseed: "137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dedb3bc0a9ac6c79b9c426c5878ca2c9d06ff42a23cb648312fc32ba83649de0928e066137103c28caeebbcea5d7f95edb97a289ded151b72159137cb7b2671f394f54cff8c121589dcb373e267225547b3c71cbdb54f6e48ec85cd549f96cf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0dcf0d".into(), address: H160::from_str("ede84640d1a1d3e06902048e67aa7db8d52c2ce1").unwrap(), }; diff --git a/ethstore/tests/api.rs b/ethstore/tests/api.rs index 83aa04874..e1667607b 100644 --- a/ethstore/tests/api.rs +++ b/ethstore/tests/api.rs @@ -19,9 +19,8 @@ extern crate ethstore; mod util; -use std::str::FromStr; use ethstore::{SecretStore, EthStore}; -use ethstore::ethkey::{Random, Generator, Secret, Address}; +use ethstore::ethkey::{Random, Generator, Secret, KeyPair, verify_address}; use ethstore::dir::DiskDirectory; use util::TransientDir; @@ -103,14 +102,21 @@ fn pat_path() -> &'static str { } } +fn ciphertext_path() -> &'static str { + match ::std::fs::metadata("ethstore") { + Ok(_) => "ethstore/tests/res/ciphertext", + Err(_) => "tests/res/ciphertext", + } +} + #[test] fn secret_store_laod_geth_files() { let dir = DiskDirectory::at(test_path()); let store = EthStore::open(Box::new(dir)).unwrap(); assert_eq!(store.accounts().unwrap(), vec![ - Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap(), - Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap(), - Address::from_str("63121b431a52f8043c16fcf0d1df9cb7b5f66649").unwrap(), + "3f49624084b67849c7b4e805c5988c21a430f9d9".into(), + "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf".into(), + "63121b431a52f8043c16fcf0d1df9cb7b5f66649".into(), ]); } @@ -119,9 +125,30 @@ fn secret_store_load_pat_files() { let dir = DiskDirectory::at(pat_path()); let store = EthStore::open(Box::new(dir)).unwrap(); assert_eq!(store.accounts().unwrap(), vec![ - Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap(), - Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap(), + "3f49624084b67849c7b4e805c5988c21a430f9d9".into(), + "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf".into(), ]); } +#[test] +fn test_decrypting_files_with_short_ciphertext() { + // 31e9d1e6d844bd3a536800ef8d8be6a9975db509, 30 + let kp1 = KeyPair::from_secret("000081c29e8142bb6a81bef5a92bda7a8328a5c85bb2f9542e76f9b0f94fc018".into()).unwrap(); + // d1e64e5480bfaf733ba7d48712decb8227797a4e , 31 + let kp2 = KeyPair::from_secret("00fa7b3db73dc7dfdf8c5fbdb796d741e4488628c41fc4febd9160a866ba0f35".into()).unwrap(); + let dir = DiskDirectory::at(ciphertext_path()); + let store = EthStore::open(Box::new(dir)).unwrap(); + let accounts = store.accounts().unwrap(); + assert_eq!(accounts, vec![ + "31e9d1e6d844bd3a536800ef8d8be6a9975db509".into(), + "d1e64e5480bfaf733ba7d48712decb8227797a4e".into(), + ]); + let message = Default::default(); + + let s1 = store.sign(&accounts[0], "foo", &message).unwrap(); + let s2 = store.sign(&accounts[1], "foo", &message).unwrap(); + assert!(verify_address(&accounts[0], &s1, &message).unwrap()); + assert!(verify_address(&kp1.address(), &s1, &message).unwrap()); + assert!(verify_address(&kp2.address(), &s2, &message).unwrap()); +} diff --git a/ethstore/tests/res/ciphertext/30.json b/ethstore/tests/res/ciphertext/30.json new file mode 100644 index 000000000..c4f5ad184 --- /dev/null +++ b/ethstore/tests/res/ciphertext/30.json @@ -0,0 +1,21 @@ +{ + "address" : "31e9d1e6d844bd3a536800ef8d8be6a9975db509", + "crypto" : { + "cipher" : "aes-128-ctr", + "cipherparams" : { + "iv" : "3ca92af36ad7c2cd92454c59cea5ef00" + }, + "ciphertext" : "108b7d34f3442fc26ab1ab90ca91476ba6bfa8c00975a49ef9051dc675aa", + "kdf" : "scrypt", + "kdfparams" : { + "dklen" : 32, + "n" : 2, + "r" : 8, + "p" : 1, + "salt" : "d0769e608fb86cda848065642a9c6fa046845c928175662b8e356c77f914cd3b" + }, + "mac" : "75d0e6759f7b3cefa319c3be41680ab6beea7d8328653474bd06706d4cc67420" + }, + "id" : "a37e1559-5955-450d-8075-7b8931b392b2", + "version" : 3 +} diff --git a/ethstore/tests/res/ciphertext/31.json b/ethstore/tests/res/ciphertext/31.json new file mode 100644 index 000000000..9c2612b03 --- /dev/null +++ b/ethstore/tests/res/ciphertext/31.json @@ -0,0 +1,21 @@ +{ + "address" : "d1e64e5480bfaf733ba7d48712decb8227797a4e", + "crypto" : { + "cipher" : "aes-128-ctr", + "cipherparams" : { + "iv" : "e0c41130a323adc1446fc82f724bca2f" + }, + "ciphertext" : "9517cd5bdbe69076f9bf5057248c6c050141e970efa36ce53692d5d59a3984", + "kdf" : "scrypt", + "kdfparams" : { + "dklen" : 32, + "n" : 2, + "r" : 8, + "p" : 1, + "salt" : "711f816911c92d649fb4c84b047915679933555030b3552c1212609b38208c63" + }, + "mac" : "d5e116151c6aa71470e67a7d42c9620c75c4d23229847dcc127794f0732b0db5" + }, + "id" : "fecfc4ce-e956-48fd-953b-30f8b52ed66c", + "version" : 3 +} diff --git a/json/src/spec/ethash.rs b/json/src/spec/ethash.rs index 10d4f84ee..d20ab3992 100644 --- a/json/src/spec/ethash.rs +++ b/json/src/spec/ethash.rs @@ -32,6 +32,9 @@ pub struct EthashParams { #[serde(rename="difficultyBoundDivisor")] pub difficulty_bound_divisor: Uint, /// See main EthashParams docs. + #[serde(rename="difficultyIncrementDivisor")] + pub difficulty_increment_divisor: Option, + /// See main EthashParams docs. #[serde(rename="durationLimit")] pub duration_limit: Uint, /// See main EthashParams docs. @@ -39,9 +42,11 @@ pub struct EthashParams { pub block_reward: Uint, /// See main EthashParams docs. pub registrar: Option
, + /// See main EthashParams docs. #[serde(rename="frontierCompatibilityModeLimit")] pub frontier_compatibility_mode_limit: Option, + /// See main EthashParams docs. #[serde(rename="daoHardforkTransition")] pub dao_hardfork_transition: Option, @@ -51,6 +56,16 @@ pub struct EthashParams { /// See main EthashParams docs. #[serde(rename="daoHardforkAccounts")] pub dao_hardfork_accounts: Option>, + + /// See main EthashParams docs. + #[serde(rename="difficultyHardforkTransition")] + pub difficulty_hardfork_transition: Option, + /// See main EthashParams docs. + #[serde(rename="difficultyHardforkBoundDivisor")] + pub difficulty_hardfork_bound_divisor: Option, + /// See main EthashParams docs. + #[serde(rename="bombDefuseTransition")] + pub bomb_defuse_transition: Option, } /// Ethash engine deserialization. @@ -99,7 +114,10 @@ mod tests { "0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97", "0xbb9bc244d798123fde783fcc1c72d3bb8c189413", "0x807640a13483f8ac783c557fcdf27be11ea4ac7a" - ] + ], + "difficultyHardforkTransition": "0x59d9", + "difficultyHardforkBoundDivisor": "0x0200", + "bombDefuseTransition": "0x42" } }"#; diff --git a/json/src/spec/params.rs b/json/src/spec/params.rs index 62c63d6b5..676feff93 100644 --- a/json/src/spec/params.rs +++ b/json/src/spec/params.rs @@ -31,6 +31,9 @@ pub struct Params { /// Network id. #[serde(rename="networkID")] pub network_id: Uint, + /// Name of the main ("eth") subprotocol. + #[serde(rename="subprotocolName")] + pub subprotocol_name: Option, /// Minimum gas limit. #[serde(rename="minGasLimit")] pub min_gas_limit: Uint, @@ -53,6 +56,7 @@ mod tests { "frontierCompatibilityModeLimit": "0x118c30", "maximumExtraDataSize": "0x20", "networkID" : "0x1", + "subprotocolName" : "exp", "minGasLimit": "0x1388", "accountStartNonce": "0x00" }"#; diff --git a/logger/src/lib.rs b/logger/src/lib.rs index e672a3e28..79655d2f6 100644 --- a/logger/src/lib.rs +++ b/logger/src/lib.rs @@ -91,10 +91,10 @@ pub fn setup_log(config: &Config) -> Result, String> { let timestamp = time::strftime("%Y-%m-%d %H:%M:%S %Z", &time::now()).unwrap(); let with_color = if max_log_level() <= LogLevelFilter::Info { - format!("{}{}", Colour::Black.bold().paint(timestamp), record.args()) + format!("{} {}", Colour::Black.bold().paint(timestamp), record.args()) } else { let name = thread::current().name().map_or_else(Default::default, |x| format!("{}", Colour::Blue.bold().paint(x))); - format!("{}{} {} {} {}", Colour::Black.bold().paint(timestamp), name, record.level(), record.target(), record.args()) + format!("{} {} {} {} {}", Colour::Black.bold().paint(timestamp), name, record.level(), record.target(), record.args()) }; let removed_color = kill_color(with_color.as_ref()); diff --git a/parity/cli/usage.txt b/parity/cli/usage.txt index a94f55a8d..4c5d3b94b 100644 --- a/parity/cli/usage.txt +++ b/parity/cli/usage.txt @@ -31,8 +31,8 @@ Operating Options: (default: {flag_mode_alarm}). --chain CHAIN Specify the blockchain type. CHAIN may be either a JSON chain specification file or olympic, frontier, - homestead, mainnet, morden, classic or testnet - (default: {flag_chain}). + homestead, mainnet, morden, classic, expanse or + testnet (default: {flag_chain}). -d --db-path PATH Specify the database & configuration directory path (default: {flag_db_path}). --keys-path PATH Specify the path for JSON key files to be found diff --git a/parity/migration.rs b/parity/migration.rs index ac96d0864..66e1d8010 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -43,13 +43,15 @@ pub enum Error { /// Returned when current version cannot be read or guessed. UnknownDatabaseVersion, /// Migration does not support existing pruning algorithm. - UnsuportedPruningMethod, + UnsupportedPruningMethod, /// Existing DB is newer than the known one. FutureDBVersion, /// Migration is not possible. MigrationImpossible, /// Migration unexpectadly failed. MigrationFailed, + /// Internal migration error. + Internal(MigrationError), /// Migration was completed succesfully, /// but there was a problem with io. Io(IoError), @@ -59,10 +61,11 @@ impl Display for Error { fn fmt(&self, f: &mut Formatter) -> Result<(), FmtError> { let out = match *self { Error::UnknownDatabaseVersion => "Current database version cannot be read".into(), - Error::UnsuportedPruningMethod => "Unsupported pruning method for database migration. Delete DB and resync.".into(), + Error::UnsupportedPruningMethod => "Unsupported pruning method for database migration. Delete DB and resync.".into(), Error::FutureDBVersion => "Database was created with newer client version. Upgrade your client or delete DB and resync.".into(), Error::MigrationImpossible => format!("Database migration to version {} is not possible.", CURRENT_VERSION), Error::MigrationFailed => "Database migration unexpectedly failed".into(), + Error::Internal(ref err) => format!("{}", err), Error::Io(ref err) => format!("Unexpected io error on DB migration: {}.", err), }; @@ -80,7 +83,7 @@ impl From for Error { fn from(err: MigrationError) -> Self { match err { MigrationError::Io(e) => Error::Io(e), - _ => Error::MigrationFailed, + _ => Error::Internal(err), } } } @@ -160,7 +163,7 @@ fn consolidate_database( let config = default_migration_settings(compaction_profile); let mut db_config = DatabaseConfig { max_open_files: 64, - cache_size: None, + cache_sizes: Default::default(), compaction: config.compaction_profile, columns: None, wal: true, @@ -320,7 +323,7 @@ mod legacy { let res = match pruning { Algorithm::Archive => manager.add_migration(migrations::state::ArchiveV7::default()), Algorithm::OverlayRecent => manager.add_migration(migrations::state::OverlayRecentV7::default()), - _ => return Err(Error::UnsuportedPruningMethod), + _ => return Err(Error::UnsupportedPruningMethod), }; try!(res.map_err(|_| Error::MigrationImpossible)); diff --git a/parity/params.rs b/parity/params.rs index 71f702cfb..160b50866 100644 --- a/parity/params.rs +++ b/parity/params.rs @@ -29,6 +29,7 @@ pub enum SpecType { Testnet, Olympic, Classic, + Expanse, Custom(String), } @@ -47,6 +48,7 @@ impl str::FromStr for SpecType { "frontier-dogmatic" | "homestead-dogmatic" | "classic" => SpecType::Classic, "morden" | "testnet" => SpecType::Testnet, "olympic" => SpecType::Olympic, + "expanse" => SpecType::Expanse, other => SpecType::Custom(other.into()), }; Ok(spec) @@ -60,6 +62,7 @@ impl SpecType { SpecType::Testnet => Ok(ethereum::new_morden()), SpecType::Olympic => Ok(ethereum::new_olympic()), SpecType::Classic => Ok(ethereum::new_classic()), + SpecType::Expanse => Ok(ethereum::new_expanse()), SpecType::Custom(ref filename) => { let file = try!(fs::File::open(filename).map_err(|_| "Could not load specification file.")); Spec::load(file) diff --git a/parity/run.rs b/parity/run.rs index e95b5c9f5..4b458d4a6 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -148,6 +148,11 @@ pub fn execute(cmd: RunCmd) -> Result<(), String> { Some(id) => id, None => spec.network_id(), }; + if spec.subprotocol_name().len() != 3 { + warn!("Your chain specification's subprotocol length is not 3. Ignoring."); + } else { + sync_config.subprotocol_name.clone_from_slice(spec.subprotocol_name().as_bytes()); + } sync_config.fork_block = spec.fork_block(); // prepare account provider diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c3f9cddbd..34b68fb81 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -25,6 +25,7 @@ ethsync = { path = "../sync" } ethjson = { path = "../json" } ethcore-devtools = { path = "../devtools" } rlp = { path = "../util/rlp" } +fetch = { path = "../util/fetch" } rustc-serialize = "0.3" transient-hashmap = "0.1" serde_macros = { version = "0.8.0", optional = true } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 7f2f11400..01ba44941 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -36,6 +36,7 @@ extern crate json_ipc_server as ipc; extern crate ethcore_ipc; extern crate time; extern crate rlp; +extern crate fetch; #[macro_use] extern crate log; diff --git a/rpc/src/v1/helpers/errors.rs b/rpc/src/v1/helpers/errors.rs index 18e369208..885ec08f0 100644 --- a/rpc/src/v1/helpers/errors.rs +++ b/rpc/src/v1/helpers/errors.rs @@ -23,6 +23,7 @@ macro_rules! rpc_unimplemented { use std::fmt; use ethcore::error::Error as EthcoreError; use ethcore::account_provider::{Error as AccountError}; +use fetch::FetchError; use jsonrpc_core::{Error, ErrorCode, Value}; mod codes { @@ -41,6 +42,7 @@ mod codes { pub const REQUEST_REJECTED_LIMIT: i64 = -32041; pub const REQUEST_NOT_FOUND: i64 = -32042; pub const COMPILATION_ERROR: i64 = -32050; + pub const FETCH_ERROR: i64 = -32060; } pub fn unimplemented() -> Error { @@ -155,6 +157,14 @@ pub fn signer_disabled() -> Error { } } +pub fn from_fetch_error(error: FetchError) -> Error { + Error { + code: ErrorCode::ServerError(codes::FETCH_ERROR), + message: "Error while fetching content.".into(), + data: Some(Value::String(format!("{:?}", error))), + } +} + pub fn from_signing_error(error: AccountError) -> Error { Error { code: ErrorCode::ServerError(codes::ACCOUNT_LOCKED), diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 755539ebd..b174e406e 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -424,13 +424,9 @@ impl Eth for EthClient where fn transaction_by_hash(&self, hash: RpcH256) -> Result, Error> { try!(self.active()); - - let miner = take_weak!(self.miner); let hash: H256 = hash.into(); - match miner.transaction(&hash) { - Some(pending_tx) => Ok(Some(pending_tx.into())), - None => self.transaction(TransactionID::Hash(hash)) - } + let miner = take_weak!(self.miner); + Ok(try!(self.transaction(TransactionID::Hash(hash))).or_else(|| miner.transaction(&hash).map(Into::into))) } fn transaction_by_block_hash_and_index(&self, hash: RpcH256, index: Index) -> Result, Error> { diff --git a/rpc/src/v1/impls/ethcore.rs b/rpc/src/v1/impls/ethcore.rs index 220ead3dd..684ce61a4 100644 --- a/rpc/src/v1/impls/ethcore.rs +++ b/rpc/src/v1/impls/ethcore.rs @@ -15,30 +15,33 @@ // along with Parity. If not, see . //! Ethcore-specific rpc implementation. -use std::sync::{Arc, Weak}; +use std::{fs, io}; +use std::sync::{mpsc, Arc, Weak}; use std::str::FromStr; use std::collections::{BTreeMap}; -use util::{RotatingLogger, Address}; +use util::{RotatingLogger, Address, Mutex, sha3}; use util::misc::version_data; use crypto::ecies; +use fetch::{Client as FetchClient, Fetch}; use ethkey::{Brain, Generator}; use ethstore::random_phrase; use ethsync::{SyncProvider, ManageNetwork}; use ethcore::miner::MinerService; use ethcore::client::{MiningBlockChainClient}; -use jsonrpc_core::*; +use jsonrpc_core::{from_params, to_value, Value, Error, Params, Ready}; use v1::traits::Ethcore; -use v1::types::{Bytes, U256, H160, H512, Peers, Transaction}; +use v1::types::{Bytes, U256, H160, H256, H512, Peers, Transaction}; use v1::helpers::{errors, SigningQueue, SignerService, NetworkSettings}; use v1::helpers::params::expect_no_params; /// Ethcore implementation. -pub struct EthcoreClient where +pub struct EthcoreClient where C: MiningBlockChainClient, M: MinerService, - S: SyncProvider { + S: SyncProvider, + F: Fetch { client: Weak, miner: Weak, @@ -47,10 +50,14 @@ pub struct EthcoreClient where logger: Arc, settings: Arc, signer: Option>, + fetch: Mutex } -impl EthcoreClient where C: MiningBlockChainClient, M: MinerService, S: SyncProvider { - /// Creates new `EthcoreClient`. +impl EthcoreClient where + C: MiningBlockChainClient, + M: MinerService, + S: SyncProvider, { + /// Creates new `EthcoreClient` with default `Fetch`. pub fn new( client: &Arc, miner: &Arc, @@ -60,6 +67,26 @@ impl EthcoreClient where C: MiningBlockChainClient, M: settings: Arc, signer: Option> ) -> Self { + Self::with_fetch(client, miner, sync, net, logger, settings, signer) + } +} + +impl EthcoreClient where + C: MiningBlockChainClient, + M: MinerService, + S: SyncProvider, + F: Fetch, { + + /// Creates new `EthcoreClient` with customizable `Fetch`. + pub fn with_fetch( + client: &Arc, + miner: &Arc, + sync: &Arc, + net: &Arc, + logger: Arc, + settings: Arc, + signer: Option> + ) -> Self { EthcoreClient { client: Arc::downgrade(client), miner: Arc::downgrade(miner), @@ -68,6 +95,7 @@ impl EthcoreClient where C: MiningBlockChainClient, M: logger: logger, settings: settings, signer: signer, + fetch: Mutex::new(F::default()), } } @@ -78,7 +106,11 @@ impl EthcoreClient where C: MiningBlockChainClient, M: } } -impl Ethcore for EthcoreClient where M: MinerService + 'static, C: MiningBlockChainClient + 'static, S: SyncProvider + 'static { +impl Ethcore for EthcoreClient where + M: MinerService + 'static, + C: MiningBlockChainClient + 'static, + S: SyncProvider + 'static, + F: Fetch + 'static { fn transactions_limit(&self, params: Params) -> Result { try!(self.active()); @@ -233,4 +265,42 @@ impl Ethcore for EthcoreClient where M: MinerService + Ok(to_value(&take_weak!(self.miner).all_transactions().into_iter().map(Into::into).collect::>())) } + + fn hash_content(&self, params: Params, ready: Ready) { + let res = self.active().and_then(|_| from_params::<(String,)>(params)); + + let hash_content = |result| { + let path = try!(result); + let mut file = io::BufReader::new(try!(fs::File::open(&path))); + // Try to hash + let result = sha3(&mut file); + // Remove file (always) + try!(fs::remove_file(&path)); + // Return the result + Ok(try!(result)) + }; + + match res { + Err(e) => ready.ready(Err(e)), + Ok((url, )) => { + let (tx, rx) = mpsc::channel(); + let res = self.fetch.lock().request_async(&url, Default::default(), Box::new(move |result| { + let result = hash_content(result) + .map_err(errors::from_fetch_error) + .map(|hash| to_value(H256::from(hash))); + + // Receive ready and invoke with result. + let ready: Ready = rx.try_recv().expect("When on_done is invoked ready object is always sent."); + ready.ready(result); + })); + + // Either invoke ready right away or transfer it to the closure. + if let Err(e) = res { + ready.ready(Err(errors::from_fetch_error(e))); + } else { + tx.send(ready).expect("Rx end is sent to on_done closure."); + } + } + } + } } diff --git a/rpc/src/v1/tests/helpers/fetch.rs b/rpc/src/v1/tests/helpers/fetch.rs new file mode 100644 index 000000000..98d888a10 --- /dev/null +++ b/rpc/src/v1/tests/helpers/fetch.rs @@ -0,0 +1,44 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Test implementation of fetch client. + +use std::io::Write; +use std::{env, fs, thread}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use fetch::{Fetch, FetchError, FetchResult}; + +/// Test implementation of fetcher. Will always return the same file. +#[derive(Default)] +pub struct TestFetch; + +impl Fetch for TestFetch { + fn request_async(&mut self, _url: &str, _abort: Arc, on_done: Box) -> Result<(), FetchError> { + thread::spawn(move || { + let mut path = env::temp_dir(); + path.push(Self::random_filename()); + + let mut file = fs::File::create(&path).unwrap(); + file.write_all(b"Some content").unwrap(); + + on_done(Ok(path)); + }); + Ok(()) + } +} + + diff --git a/rpc/src/v1/tests/helpers/mod.rs b/rpc/src/v1/tests/helpers/mod.rs index 1b8f9e256..234bae1be 100644 --- a/rpc/src/v1/tests/helpers/mod.rs +++ b/rpc/src/v1/tests/helpers/mod.rs @@ -18,6 +18,8 @@ mod sync_provider; mod miner_service; +mod fetch; pub use self::sync_provider::{Config, TestSyncProvider}; pub use self::miner_service::TestMinerService; +pub use self::fetch::TestFetch; diff --git a/rpc/src/v1/tests/mocked/ethcore.rs b/rpc/src/v1/tests/mocked/ethcore.rs index 811ccced4..3dc02e929 100644 --- a/rpc/src/v1/tests/mocked/ethcore.rs +++ b/rpc/src/v1/tests/mocked/ethcore.rs @@ -23,7 +23,7 @@ use ethcore::client::{TestBlockChainClient}; use jsonrpc_core::IoHandler; use v1::{Ethcore, EthcoreClient}; use v1::helpers::{SignerService, NetworkSettings}; -use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService}; +use v1::tests::helpers::{TestSyncProvider, Config, TestMinerService, TestFetch}; use super::manage_network::TestManageNetwork; fn miner_service() -> Arc { @@ -60,12 +60,15 @@ fn network_service() -> Arc { Arc::new(TestManageNetwork) } +type TestEthcoreClient = EthcoreClient; + fn ethcore_client( client: &Arc, miner: &Arc, sync: &Arc, - net: &Arc) -> EthcoreClient { - EthcoreClient::new(client, miner, sync, net, logger(), settings(), None) + net: &Arc) + -> TestEthcoreClient { + EthcoreClient::with_fetch(client, miner, sync, net, logger(), settings(), None) } #[test] @@ -140,9 +143,9 @@ fn rpc_ethcore_dev_logs() { let logger = logger(); logger.append("a".to_owned()); logger.append("b".to_owned()); - let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger.clone(), settings(), None).to_delegate(); + let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger.clone(), settings(), None); let io = IoHandler::new(); - io.add_delegate(ethcore); + io.add_delegate(ethcore.to_delegate()); let request = r#"{"jsonrpc": "2.0", "method": "ethcore_devLogs", "params":[], "id": 1}"#; let response = r#"{"jsonrpc":"2.0","result":["b","a"],"id":1}"#; @@ -263,8 +266,8 @@ fn rpc_ethcore_unsigned_transactions_count() { let net = network_service(); let io = IoHandler::new(); let signer = Arc::new(SignerService::new_test()); - let ethcore = EthcoreClient::new(&client, &miner, &sync, &net, logger(), settings(), Some(signer)).to_delegate(); - io.add_delegate(ethcore); + let ethcore: TestEthcoreClient = EthcoreClient::with_fetch(&client, &miner, &sync, &net, logger(), settings(), Some(signer)); + io.add_delegate(ethcore.to_delegate()); let request = r#"{"jsonrpc": "2.0", "method": "ethcore_unsignedTransactionsCount", "params":[], "id": 1}"#; let response = r#"{"jsonrpc":"2.0","result":0,"id":1}"#; @@ -287,6 +290,21 @@ fn rpc_ethcore_unsigned_transactions_count_when_signer_disabled() { assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); } +#[test] +fn rpc_ethcore_hash_content() { + let miner = miner_service(); + let client = client_service(); + let sync = sync_provider(); + let net = network_service(); + let io = IoHandler::new(); + io.add_delegate(ethcore_client(&client, &miner, &sync, &net).to_delegate()); + + let request = r#"{"jsonrpc": "2.0", "method": "ethcore_hashContent", "params":["https://ethcore.io/assets/images/ethcore-black-horizontal.png"], "id": 1}"#; + let response = r#"{"jsonrpc":"2.0","result":"0x2be00befcf008bc0e7d9cdefc194db9c75352e8632f48498b5a6bfce9f02c88e","id":1}"#; + + assert_eq!(io.handle_request_sync(request), Some(response.to_owned())); +} + #[test] fn rpc_ethcore_pending_transactions() { let miner = miner_service(); diff --git a/rpc/src/v1/traits/ethcore.rs b/rpc/src/v1/traits/ethcore.rs index 56c27534a..0565da04a 100644 --- a/rpc/src/v1/traits/ethcore.rs +++ b/rpc/src/v1/traits/ethcore.rs @@ -83,6 +83,9 @@ pub trait Ethcore: Sized + Send + Sync + 'static { /// Returns all pending (current) transactions from transaction queue. fn pending_transactions(&self, _: Params) -> Result; + /// Hash a file content under given URL. + fn hash_content(&self, _: Params, _: Ready); + /// Should be used to convert object to io delegate. fn to_delegate(self) -> IoDelegate { let mut delegate = IoDelegate::new(Arc::new(self)); @@ -107,6 +110,8 @@ pub trait Ethcore: Sized + Send + Sync + 'static { delegate.add_method("ethcore_registryAddress", Ethcore::registry_address); delegate.add_method("ethcore_encryptMessage", Ethcore::encrypt_message); delegate.add_method("ethcore_pendingTransactions", Ethcore::pending_transactions); + delegate.add_async_method("ethcore_hashContent", Ethcore::hash_content); + delegate } } diff --git a/rpc/src/v1/types/hash.rs b/rpc/src/v1/types/hash.rs index 3080aa031..3db0cf124 100644 --- a/rpc/src/v1/types/hash.rs +++ b/rpc/src/v1/types/hash.rs @@ -25,9 +25,10 @@ use util::{H64 as Eth64, H160 as Eth160, H256 as Eth256, H520 as Eth520, H512 as macro_rules! impl_hash { ($name: ident, $other: ident, $size: expr) => { /// Hash serialization - #[derive(Eq)] pub struct $name([u8; $size]); + impl Eq for $name { } + impl Default for $name { fn default() -> Self { $name([0; $size]) diff --git a/rpc/src/v1/types/trace.rs b/rpc/src/v1/types/trace.rs index f66d8e0c1..ace76827c 100644 --- a/rpc/src/v1/types/trace.rs +++ b/rpc/src/v1/types/trace.rs @@ -414,15 +414,15 @@ pub struct LocalizedTrace { /// Result result: Res, /// Trace address - trace_address: Vec, + trace_address: Vec, /// Subtraces - subtraces: U256, + subtraces: usize, /// Transaction position - transaction_position: U256, + transaction_position: usize, /// Transaction hash transaction_hash: H256, /// Block Number - block_number: U256, + block_number: u64, /// Block Hash block_hash: H256, } @@ -485,9 +485,9 @@ impl From for LocalizedTrace { #[derive(Debug)] pub struct Trace { /// Trace address - trace_address: Vec, + trace_address: Vec, /// Subtraces - subtraces: U256, + subtraces: usize, /// Action action: Action, /// Result @@ -601,15 +601,15 @@ mod tests { gas_used: 8.into(), output: vec![0x56, 0x78].into(), }), - trace_address: vec![10.into()], - subtraces: 1.into(), - transaction_position: 11.into(), + trace_address: vec![10], + subtraces: 1, + transaction_position: 11, transaction_hash: 12.into(), - block_number: 13.into(), + block_number: 13, block_hash: 14.into(), }; let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"type":"call","action":{"from":"0x0000000000000000000000000000000000000004","to":"0x0000000000000000000000000000000000000005","value":"0x6","gas":"0x7","input":"0x1234","callType":"call"},"result":{"gasUsed":"0x8","output":"0x5678"},"traceAddress":["0xa"],"subtraces":"0x1","transactionPosition":"0xb","transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":"0xd","blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); + assert_eq!(serialized, r#"{"type":"call","action":{"from":"0x0000000000000000000000000000000000000004","to":"0x0000000000000000000000000000000000000005","value":"0x6","gas":"0x7","input":"0x1234","callType":"call"},"result":{"gasUsed":"0x8","output":"0x5678"},"traceAddress":[10],"subtraces":1,"transactionPosition":11,"transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":13,"blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); } #[test] @@ -624,15 +624,15 @@ mod tests { call_type: CallType::Call, }), result: Res::FailedCall(TraceError::OutOfGas), - trace_address: vec![10.into()], - subtraces: 1.into(), - transaction_position: 11.into(), + trace_address: vec![10], + subtraces: 1, + transaction_position: 11, transaction_hash: 12.into(), - block_number: 13.into(), + block_number: 13, block_hash: 14.into(), }; let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"type":"call","action":{"from":"0x0000000000000000000000000000000000000004","to":"0x0000000000000000000000000000000000000005","value":"0x6","gas":"0x7","input":"0x1234","callType":"call"},"error":"Out of gas","traceAddress":["0xa"],"subtraces":"0x1","transactionPosition":"0xb","transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":"0xd","blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); + assert_eq!(serialized, r#"{"type":"call","action":{"from":"0x0000000000000000000000000000000000000004","to":"0x0000000000000000000000000000000000000005","value":"0x6","gas":"0x7","input":"0x1234","callType":"call"},"error":"Out of gas","traceAddress":[10],"subtraces":1,"transactionPosition":11,"transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":13,"blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); } #[test] @@ -649,15 +649,15 @@ mod tests { code: vec![0x56, 0x78].into(), address: 0xff.into(), }), - trace_address: vec![10.into()], - subtraces: 1.into(), - transaction_position: 11.into(), + trace_address: vec![10], + subtraces: 1, + transaction_position: 11, transaction_hash: 12.into(), - block_number: 13.into(), + block_number: 13, block_hash: 14.into(), }; let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"type":"create","action":{"from":"0x0000000000000000000000000000000000000004","value":"0x6","gas":"0x7","init":"0x1234"},"result":{"gasUsed":"0x8","code":"0x5678","address":"0x00000000000000000000000000000000000000ff"},"traceAddress":["0xa"],"subtraces":"0x1","transactionPosition":"0xb","transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":"0xd","blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); + assert_eq!(serialized, r#"{"type":"create","action":{"from":"0x0000000000000000000000000000000000000004","value":"0x6","gas":"0x7","init":"0x1234"},"result":{"gasUsed":"0x8","code":"0x5678","address":"0x00000000000000000000000000000000000000ff"},"traceAddress":[10],"subtraces":1,"transactionPosition":11,"transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":13,"blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); } #[test] @@ -670,15 +670,15 @@ mod tests { init: Bytes::new(vec![0x12, 0x34]), }), result: Res::FailedCreate(TraceError::OutOfGas), - trace_address: vec![10.into()], - subtraces: 1.into(), - transaction_position: 11.into(), + trace_address: vec![10], + subtraces: 1, + transaction_position: 11, transaction_hash: 12.into(), - block_number: 13.into(), + block_number: 13, block_hash: 14.into(), }; let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"type":"create","action":{"from":"0x0000000000000000000000000000000000000004","value":"0x6","gas":"0x7","init":"0x1234"},"error":"Out of gas","traceAddress":["0xa"],"subtraces":"0x1","transactionPosition":"0xb","transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":"0xd","blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); + assert_eq!(serialized, r#"{"type":"create","action":{"from":"0x0000000000000000000000000000000000000004","value":"0x6","gas":"0x7","init":"0x1234"},"error":"Out of gas","traceAddress":[10],"subtraces":1,"transactionPosition":11,"transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":13,"blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); } #[test] @@ -690,15 +690,15 @@ mod tests { balance: 7.into(), }), result: Res::None, - trace_address: vec![10.into()], - subtraces: 1.into(), - transaction_position: 11.into(), + trace_address: vec![10], + subtraces: 1, + transaction_position: 11, transaction_hash: 12.into(), - block_number: 13.into(), + block_number: 13, block_hash: 14.into(), }; let serialized = serde_json::to_string(&t).unwrap(); - assert_eq!(serialized, r#"{"type":"suicide","action":{"address":"0x0000000000000000000000000000000000000004","refundAddress":"0x0000000000000000000000000000000000000006","balance":"0x7"},"result":null,"traceAddress":["0xa"],"subtraces":"0x1","transactionPosition":"0xb","transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":"0xd","blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); + assert_eq!(serialized, r#"{"type":"suicide","action":{"address":"0x0000000000000000000000000000000000000004","refundAddress":"0x0000000000000000000000000000000000000006","balance":"0x7"},"result":null,"traceAddress":[10],"subtraces":1,"transactionPosition":11,"transactionHash":"0x000000000000000000000000000000000000000000000000000000000000000c","blockNumber":13,"blockHash":"0x000000000000000000000000000000000000000000000000000000000000000e"}"#); } #[test] diff --git a/rpc/src/v1/types/uint.rs b/rpc/src/v1/types/uint.rs index 9be7b1170..ce0fa49a2 100644 --- a/rpc/src/v1/types/uint.rs +++ b/rpc/src/v1/types/uint.rs @@ -23,9 +23,11 @@ use util::{U256 as EthU256, Uint}; macro_rules! impl_uint { ($name: ident, $other: ident, $size: expr) => { /// Uint serialization. - #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] + #[derive(Debug, Default, Clone, Copy, PartialEq, Hash)] pub struct $name($other); + impl Eq for $name { } + impl From for $name where $other: From { fn from(o: T) -> Self { $name($other::from(o)) diff --git a/sync/src/api.rs b/sync/src/api.rs index 721b29ef8..9cafc8f93 100644 --- a/sync/src/api.rs +++ b/sync/src/api.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use std::sync::Arc; +use std::str; use network::{NetworkProtocolHandler, NetworkService, NetworkContext, PeerId, NetworkConfiguration as BasicNetworkConfiguration, NonReservedPeerMode, NetworkError}; use util::{U256, H256}; @@ -31,9 +32,9 @@ use std::str::FromStr; use parking_lot::RwLock; /// Ethereum sync protocol -pub const ETH_PROTOCOL: &'static str = "eth"; +pub const ETH_PROTOCOL: [u8; 3] = *b"eth"; /// Infinity protocol -pub const INF_PROTOCOL: &'static str = "inf"; +pub const INF_PROTOCOL: [u8; 3] = *b"inf"; /// Sync configuration #[derive(Debug, Clone, Copy)] @@ -42,6 +43,8 @@ pub struct SyncConfig { pub max_download_ahead_blocks: usize, /// Network ID pub network_id: U256, + /// Main "eth" subprotocol name. + pub subprotocol_name: [u8; 3], /// Fork block to check pub fork_block: Option<(BlockNumber, H256)>, } @@ -51,6 +54,7 @@ impl Default for SyncConfig { SyncConfig { max_download_ahead_blocks: 20000, network_id: U256::from(1), + subprotocol_name: ETH_PROTOCOL, fork_block: None, } } @@ -73,6 +77,8 @@ pub struct EthSync { eth_handler: Arc, /// Infinity Protocol handler inf_handler: Arc, + /// The main subprotocol name + subprotocol_name: [u8; 3], } impl EthSync { @@ -85,6 +91,7 @@ impl EthSync { network: service, eth_handler: Arc::new(SyncProtocolHandler { sync: RwLock::new(chain_sync), chain: chain.clone(), snapshot_service: snapshot_service.clone() }), inf_handler: Arc::new(InfProtocolHandler { sync: RwLock::new(inf_sync), chain: chain, snapshot_service: snapshot_service }), + subprotocol_name: config.subprotocol_name, }); Ok(sync) @@ -171,7 +178,7 @@ impl ChainNotify for EthSync { sealed: Vec, _duration: u64) { - self.network.with_context(ETH_PROTOCOL, |context| { + self.network.with_context(self.subprotocol_name, |context| { let mut sync_io = NetSyncIo::new(context, &*self.eth_handler.chain, &*self.eth_handler.snapshot_service); self.eth_handler.sync.write().chain_new_blocks( &mut sync_io, @@ -185,7 +192,7 @@ impl ChainNotify for EthSync { fn start(&self) { self.network.start().unwrap_or_else(|e| warn!("Error starting network: {:?}", e)); - self.network.register_protocol(self.eth_handler.clone(), ETH_PROTOCOL, &[62u8, 63u8, 64u8]) + self.network.register_protocol(self.eth_handler.clone(), self.subprotocol_name, &[62u8, 63u8, 64u8]) .unwrap_or_else(|e| warn!("Error registering ethereum protocol: {:?}", e)); self.network.register_protocol(self.inf_handler.clone(), INF_PROTOCOL, &[1u8]) .unwrap_or_else(|e| warn!("Error registering infinity protocol: {:?}", e)); @@ -249,7 +256,7 @@ impl ManageNetwork for EthSync { } fn stop_network(&self) { - self.network.with_context(ETH_PROTOCOL, |context| { + self.network.with_context(self.subprotocol_name, |context| { let mut sync_io = NetSyncIo::new(context, &*self.eth_handler.chain, &*self.eth_handler.snapshot_service); self.eth_handler.sync.write().abort(&mut sync_io); }); diff --git a/sync/src/blocks.rs b/sync/src/blocks.rs index ad842ced6..beaa49c60 100644 --- a/sync/src/blocks.rs +++ b/sync/src/blocks.rs @@ -19,7 +19,7 @@ use rlp::*; use network::NetworkError; use ethcore::header::{ Header as BlockHeader}; -known_heap_size!(0, HeaderId, SyncBlock); +known_heap_size!(0, HeaderId); /// Block data with optional body. struct SyncBlock { @@ -27,6 +27,12 @@ struct SyncBlock { body: Option, } +impl HeapSizeOf for SyncBlock { + fn heap_size_of_children(&self) -> usize { + self.header.heap_size_of_children() + self.body.heap_size_of_children() + } +} + /// Used to identify header by transactions and uncles hashes #[derive(Eq, PartialEq, Hash)] struct HeaderId { @@ -219,10 +225,14 @@ impl BlockCollection { self.blocks.contains_key(hash) } - /// Return heap size. + /// Return used heap size. pub fn heap_size(&self) -> usize { - //TODO: other collections - self.blocks.heap_size_of_children() + self.heads.heap_size_of_children() + + self.blocks.heap_size_of_children() + + self.parents.heap_size_of_children() + + self.header_ids.heap_size_of_children() + + self.downloading_headers.heap_size_of_children() + + self.downloading_bodies.heap_size_of_children() } /// Check if given block hash is marked as being downloaded. diff --git a/sync/src/sync_io.rs b/sync/src/sync_io.rs index fa95941ea..445939399 100644 --- a/sync/src/sync_io.rs +++ b/sync/src/sync_io.rs @@ -17,7 +17,6 @@ use network::{NetworkContext, PeerId, PacketId, NetworkError}; use ethcore::client::BlockChainClient; use ethcore::snapshot::SnapshotService; -use api::ETH_PROTOCOL; /// IO interface for the syning handler. /// Provides peer connection management and an interface to the blockchain client. @@ -101,7 +100,7 @@ impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> { } fn eth_protocol_version(&self, peer_id: PeerId) -> u8 { - self.network.protocol_version(peer_id, ETH_PROTOCOL).unwrap_or(0) + self.network.protocol_version(peer_id, self.network.subprotocol_name()).unwrap_or(0) } } diff --git a/util/bigint/src/hash.rs b/util/bigint/src/hash.rs index 97b9545bc..f782d1f90 100644 --- a/util/bigint/src/hash.rs +++ b/util/bigint/src/hash.rs @@ -64,11 +64,11 @@ pub fn clean_0x(s: &str) -> &str { macro_rules! impl_hash { ($from: ident, $size: expr) => { - #[derive(Eq)] #[repr(C)] /// Unformatted binary data of fixed length. pub struct $from (pub [u8; $size]); + impl From<[u8; $size]> for $from { fn from(bytes: [u8; $size]) -> Self { $from(bytes) @@ -210,6 +210,8 @@ macro_rules! impl_hash { } } + impl Eq for $from {} + impl PartialEq for $from { fn eq(&self, other: &Self) -> bool { for i in 0..$size { diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml new file mode 100644 index 000000000..663d167bf --- /dev/null +++ b/util/fetch/Cargo.toml @@ -0,0 +1,18 @@ +[package] +description = "HTTP/HTTPS fetching library" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "fetch" +version = "0.1.0" +authors = ["Ethcore "] + +[dependencies] +log = "0.3" +rand = "0.3" +hyper = { default-features = false, git = "https://github.com/ethcore/hyper" } +https-fetch = { path = "../https-fetch" } +clippy = { version = "0.0.90", optional = true} + +[features] +default = [] +dev = ["clippy"] diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs new file mode 100644 index 000000000..bb8842a5b --- /dev/null +++ b/util/fetch/src/client.rs @@ -0,0 +1,146 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Fetching + +use std::{env, io}; +use std::sync::{mpsc, Arc}; +use std::sync::atomic::AtomicBool; +use std::path::PathBuf; + +use hyper; +use https_fetch as https; + +use fetch_file::{FetchHandler, Error as HttpFetchError}; + +pub type FetchResult = Result; + +#[derive(Debug)] +pub enum FetchError { + InvalidUrl, + Http(HttpFetchError), + Https(https::FetchError), + Io(io::Error), + Other(String), +} + +impl From for FetchError { + fn from(e: HttpFetchError) -> Self { + FetchError::Http(e) + } +} + +impl From for FetchError { + fn from(e: io::Error) -> Self { + FetchError::Io(e) + } +} + +pub trait Fetch: Default + Send { + /// Fetch URL and get the result in callback. + fn request_async(&mut self, url: &str, abort: Arc, on_done: Box) -> Result<(), FetchError>; + + /// Fetch URL and get a result Receiver. You will be notified when receiver is ready by `on_done` callback. + fn request(&mut self, url: &str, abort: Arc, on_done: Box) -> Result, FetchError> { + let (tx, rx) = mpsc::channel(); + try!(self.request_async(url, abort, Box::new(move |result| { + let res = tx.send(result); + if let Err(_) = res { + warn!("Fetch finished, but no one was listening"); + } + on_done(); + }))); + Ok(rx) + } + + /// Closes this client + fn close(self) {} + + /// Returns a random filename + fn random_filename() -> String { + use ::rand::Rng; + let mut rng = ::rand::OsRng::new().unwrap(); + rng.gen_ascii_chars().take(12).collect() + } +} + +pub struct Client { + http_client: hyper::Client, + https_client: https::Client, + limit: Option, +} + +impl Default for Client { + fn default() -> Self { + // Max 15MB will be downloaded. + Client::with_limit(Some(15*1024*1024)) + } +} + +impl Client { + fn with_limit(limit: Option) -> Self { + Client { + http_client: hyper::Client::new().expect("Unable to initialize http client."), + https_client: https::Client::with_limit(limit).expect("Unable to initialize https client."), + limit: limit, + } + } + + fn convert_url(url: hyper::Url) -> Result { + let host = format!("{}", try!(url.host().ok_or(FetchError::InvalidUrl))); + let port = try!(url.port_or_known_default().ok_or(FetchError::InvalidUrl)); + https::Url::new(&host, port, url.path()).map_err(|_| FetchError::InvalidUrl) + } + + fn temp_path() -> PathBuf { + let mut dir = env::temp_dir(); + dir.push(Self::random_filename()); + dir + } +} + +impl Fetch for Client { + fn close(self) { + self.http_client.close(); + self.https_client.close(); + } + + fn request_async(&mut self, url: &str, abort: Arc, on_done: Box) -> Result<(), FetchError> { + let is_https = url.starts_with("https://"); + let url = try!(url.parse().map_err(|_| FetchError::InvalidUrl)); + let temp_path = Self::temp_path(); + + trace!(target: "fetch", "Fetching from: {:?}", url); + + if is_https { + let url = try!(Self::convert_url(url)); + try!(self.https_client.fetch_to_file( + url, + temp_path.clone(), + abort, + move |result| on_done(result.map(|_| temp_path).map_err(FetchError::Https)), + ).map_err(|e| FetchError::Other(format!("{:?}", e)))); + } else { + try!(self.http_client.request( + url, + FetchHandler::new(temp_path, abort, Box::new(move |result| on_done(result)), self.limit.map(|v| v as u64).clone()), + ).map_err(|e| FetchError::Other(format!("{:?}", e)))); + } + + Ok(()) + } +} + diff --git a/dapps/src/handlers/client/fetch_file.rs b/util/fetch/src/fetch_file.rs similarity index 82% rename from dapps/src/handlers/client/fetch_file.rs rename to util/fetch/src/fetch_file.rs index c18fb6d5b..4801cc969 100644 --- a/dapps/src/handlers/client/fetch_file.rs +++ b/util/fetch/src/fetch_file.rs @@ -16,12 +16,11 @@ //! Hyper Client Handler to Fetch File -use std::{env, io, fs, fmt}; +use std::{io, fs, fmt}; use std::path::PathBuf; -use std::sync::{mpsc, Arc}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use random_filename; use hyper::status::StatusCode; use hyper::client::{Request, Response, DefaultTransport as HttpStream}; @@ -34,30 +33,31 @@ use super::FetchError; pub enum Error { Aborted, NotStarted, + SizeLimit, UnexpectedStatus(StatusCode), IoError(io::Error), HyperError(hyper::Error), } pub type FetchResult = Result; -pub type OnDone = Box; +pub type OnDone = Box; -pub struct Fetch { +pub struct FetchHandler { path: PathBuf, abort: Arc, file: Option, result: Option, - sender: mpsc::Sender, on_done: Option, + size_limit: Option, } -impl fmt::Debug for Fetch { +impl fmt::Debug for FetchHandler { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { write!(f, "Fetch {{ path: {:?}, file: {:?}, result: {:?} }}", self.path, self.file, self.result) } } -impl Drop for Fetch { +impl Drop for FetchHandler { fn drop(&mut self) { let res = self.result.take().unwrap_or(Err(Error::NotStarted.into())); // Remove file if there was an error @@ -69,40 +69,35 @@ impl Drop for Fetch { } } // send result - let _ = self.sender.send(res); if let Some(f) = self.on_done.take() { - f(); + f(res); } } } -impl Fetch { - pub fn new(sender: mpsc::Sender, abort: Arc, on_done: OnDone) -> Self { - let mut dir = env::temp_dir(); - dir.push(random_filename()); - - Fetch { - path: dir, +impl FetchHandler { + pub fn new(path: PathBuf, abort: Arc, on_done: OnDone, size_limit: Option) -> Self { + FetchHandler { + path: path, abort: abort, file: None, result: None, - sender: sender, on_done: Some(on_done), + size_limit: size_limit, } } -} -impl Fetch { fn is_aborted(&self) -> bool { self.abort.load(Ordering::SeqCst) } + fn mark_aborted(&mut self) -> Next { self.result = Some(Err(Error::Aborted.into())); Next::end() } } -impl hyper::client::Handler for Fetch { +impl hyper::client::Handler for FetchHandler { fn on_request(&mut self, req: &mut Request) -> Next { if self.is_aborted() { return self.mark_aborted(); @@ -147,7 +142,19 @@ impl hyper::client::Handler for Fetch { } match io::copy(decoder, self.file.as_mut().expect("File is there because on_response has created it.")) { Ok(0) => Next::end(), - Ok(_) => read(), + Ok(bytes_read) => match self.size_limit { + None => read(), + // Check limit + Some(limit) if limit > bytes_read => { + self.size_limit = Some(limit - bytes_read); + read() + }, + // Size limit reached + _ => { + self.result = Some(Err(Error::SizeLimit.into())); + Next::end() + }, + }, Err(e) => match e.kind() { io::ErrorKind::WouldBlock => Next::read(), _ => { diff --git a/ethcore/src/types/block_queue_info.rs b/util/fetch/src/lib.rs similarity index 55% rename from ethcore/src/types/block_queue_info.rs rename to util/fetch/src/lib.rs index d299258ce..8ec9e0ddd 100644 --- a/ethcore/src/types/block_queue_info.rs +++ b/util/fetch/src/lib.rs @@ -14,21 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Block queue info types +//! A service to fetch any HTTP / HTTPS content. -/// Block queue status -#[derive(Debug, Binary)] -pub struct BlockQueueInfo { - /// Number of queued blocks pending verification - pub unverified_queue_size: usize, - /// Number of verified queued blocks pending import - pub verified_queue_size: usize, - /// Number of blocks being verified - pub verifying_queue_size: usize, - /// Configured maximum number of blocks in the queue - pub max_queue_size: usize, - /// Configured maximum number of bytes to use - pub max_mem_use: usize, - /// Heap memory used in bytes - pub mem_used: usize, -} +#[macro_use] +extern crate log; +extern crate hyper; +extern crate https_fetch; +extern crate rand; + + +pub mod client; +pub mod fetch_file; + +pub use self::client::{Client, Fetch, FetchError, FetchResult}; diff --git a/util/https-fetch/src/client.rs b/util/https-fetch/src/client.rs index 3e5d50515..ad75f2ca4 100644 --- a/util/https-fetch/src/client.rs +++ b/util/https-fetch/src/client.rs @@ -78,6 +78,10 @@ impl Drop for Client { impl Client { pub fn new() -> Result { + Self::with_limit(None) + } + + pub fn with_limit(size_limit: Option) -> Result { let mut event_loop = try!(mio::EventLoop::new()); let channel = event_loop.channel(); @@ -85,6 +89,7 @@ impl Client { let mut client = ClientLoop { next_token: 0, sessions: HashMap::new(), + size_limit: size_limit, }; event_loop.run(&mut client).unwrap(); }); @@ -128,6 +133,7 @@ impl Client { pub struct ClientLoop { next_token: usize, sessions: HashMap, + size_limit: Option, } impl mio::Handler for ClientLoop { @@ -154,7 +160,7 @@ impl mio::Handler for ClientLoop { let token = self.next_token; self.next_token += 1; - if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback) { + if let Ok(mut tlsclient) = TlsClient::new(mio::Token(token), &url, writer, abort, callback, self.size_limit.clone()) { let httpreq = format!( "GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n", url.path(), diff --git a/util/https-fetch/src/http.rs b/util/https-fetch/src/http.rs index d29974c01..5f40ca4cb 100644 --- a/util/https-fetch/src/http.rs +++ b/util/https-fetch/src/http.rs @@ -35,18 +35,20 @@ pub struct HttpProcessor { status: Option, headers: Vec, body_writer: io::BufWriter>, + size_limit: Option, } const BREAK_LEN: usize = 2; impl HttpProcessor { - pub fn new(body_writer: Box) -> Self { + pub fn new(body_writer: Box, size_limit: Option) -> Self { HttpProcessor { state: State::WaitingForStatus, buffer: Cursor::new(Vec::new()), status: None, headers: Vec::new(), - body_writer: io::BufWriter::new(body_writer) + body_writer: io::BufWriter::new(body_writer), + size_limit: size_limit, } } @@ -140,6 +142,15 @@ impl HttpProcessor { }, State::WritingBody => { let len = self.buffer.get_ref().len(); + match self.size_limit { + None => {}, + Some(limit) if limit > len => {}, + _ => { + warn!("Finishing file fetching because limit was reached."); + self.set_state(State::Finished); + continue; + } + } try!(self.body_writer.write_all(self.buffer.get_ref())); self.buffer_consume(len); return Ok(()); @@ -167,6 +178,17 @@ impl HttpProcessor { }, // Buffers the data until we have a full chunk State::WritingChunk(left) if self.buffer.get_ref().len() >= left => { + match self.size_limit { + None => {}, + Some(limit) if limit > left => { + self.size_limit = Some(limit - left); + }, + _ => { + warn!("Finishing file fetching because limit was reached."); + self.set_state(State::Finished); + continue; + } + } try!(self.body_writer.write_all(&self.buffer.get_ref()[0..left])); self.buffer_consume(left + BREAK_LEN); @@ -230,7 +252,7 @@ mod tests { #[test] fn should_be_able_to_process_status_line() { // given - let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new()))); + let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None); // when let out = @@ -249,7 +271,7 @@ mod tests { #[test] fn should_be_able_to_process_headers() { // given - let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new()))); + let mut http = HttpProcessor::new(Box::new(Cursor::new(Vec::new())), None); // when let out = @@ -274,7 +296,7 @@ mod tests { fn should_be_able_to_consume_body() { // given let (writer, data) = Writer::new(); - let mut http = HttpProcessor::new(Box::new(writer)); + let mut http = HttpProcessor::new(Box::new(writer), None); // when let out = @@ -301,7 +323,7 @@ mod tests { fn should_correctly_handle_chunked_content() { // given let (writer, data) = Writer::new(); - let mut http = HttpProcessor::new(Box::new(writer)); + let mut http = HttpProcessor::new(Box::new(writer), None); // when let out = @@ -331,4 +353,40 @@ mod tests { assert_eq!(data.borrow().get_ref()[..], b"Parity in\r\n\r\nchunks."[..]); assert_eq!(http.state(), State::Finished); } + + #[test] + fn should_stop_fetching_when_limit_is_reached() { + // given + let (writer, data) = Writer::new(); + let mut http = HttpProcessor::new(Box::new(writer), Some(5)); + + // when + let out = + "\ + HTTP/1.1 200 OK\r\n\ + Host: 127.0.0.1:8080\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: close\r\n\ + \r\n\ + 4\r\n\ + Pari\r\n\ + 3\r\n\ + ty \r\n\ + D\r\n\ + in\r\n\ + \r\n\ + chunks.\r\n\ + 0\r\n\ + \r\n\ + "; + http.write_all(out.as_bytes()).unwrap(); + http.flush().unwrap(); + + // then + assert_eq!(http.status().unwrap(), "HTTP/1.1 200 OK"); + assert_eq!(http.headers().len(), 3); + assert_eq!(data.borrow().get_ref()[..], b"Pari"[..]); + assert_eq!(http.state(), State::Finished); + } + } diff --git a/util/https-fetch/src/tlsclient.rs b/util/https-fetch/src/tlsclient.rs index e3ce44764..62af2b06c 100644 --- a/util/https-fetch/src/tlsclient.rs +++ b/util/https-fetch/src/tlsclient.rs @@ -87,6 +87,7 @@ impl TlsClient { writer: Box, abort: Arc, mut callback: Box, + size_limit: Option, ) -> Result { let res = TlsClient::make_config().and_then(|cfg| { TcpStream::connect(url.address()).map(|sock| { @@ -98,7 +99,7 @@ impl TlsClient { Ok((cfg, sock)) => Ok(TlsClient { abort: abort, token: token, - writer: HttpProcessor::new(writer), + writer: HttpProcessor::new(writer, size_limit), socket: sock, closing: false, error: None, diff --git a/util/network/src/host.rs b/util/network/src/host.rs index ebc10324f..f530503c6 100644 --- a/util/network/src/host.rs +++ b/util/network/src/host.rs @@ -137,7 +137,7 @@ const SYS_TIMER: usize = LAST_SESSION + 1; /// Protocol handler level packet id pub type PacketId = u8; /// Protocol / handler id -pub type ProtocolId = &'static str; +pub type ProtocolId = [u8; 3]; /// Messages used to communitate with the event loop from other threads. #[derive(Clone)] @@ -185,7 +185,7 @@ pub struct CapabilityInfo { impl Encodable for CapabilityInfo { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(2); - s.append(&self.protocol); + s.append(&&self.protocol[..]); s.append(&self.version); } } @@ -284,10 +284,13 @@ impl<'s> NetworkContext<'s> { } /// Returns max version for a given protocol. - pub fn protocol_version(&self, peer: PeerId, protocol: &str) -> Option { + pub fn protocol_version(&self, peer: PeerId, protocol: ProtocolId) -> Option { let session = self.resolve_session(peer); session.and_then(|s| s.lock().capability_version(protocol)) } + + /// Returns this object's subprotocol name. + pub fn subprotocol_name(&self) -> ProtocolId { self.protocol } } /// Shared host information @@ -801,8 +804,8 @@ impl Host { } } for (p, _) in self.handlers.read().iter() { - if s.have_capability(p) { - ready_data.push(p); + if s.have_capability(*p) { + ready_data.push(*p); } } }, @@ -811,7 +814,7 @@ impl Host { protocol, packet_id, }) => { - match self.handlers.read().get(protocol) { + match self.handlers.read().get(&protocol) { None => { warn!(target: "network", "No handler found for protocol: {:?}", protocol) }, Some(_) => packet_data.push((protocol, packet_id, data)), } @@ -826,13 +829,13 @@ impl Host { } let handlers = self.handlers.read(); for p in ready_data { - let h = handlers.get(p).unwrap().clone(); + let h = handlers.get(&p).unwrap().clone(); self.stats.inc_sessions(); let reserved = self.reserved_nodes.read(); h.connected(&NetworkContext::new(io, p, session.clone(), self.sessions.clone(), &reserved), &token); } for (p, packet_id, data) in packet_data { - let h = handlers.get(p).unwrap().clone(); + let h = handlers.get(&p).unwrap().clone(); let reserved = self.reserved_nodes.read(); h.read(&NetworkContext::new(io, p, session.clone(), self.sessions.clone(), &reserved), &token, packet_id, &data[1..]); } @@ -857,8 +860,8 @@ impl Host { if s.is_ready() { self.num_sessions.fetch_sub(1, AtomicOrdering::SeqCst); for (p, _) in self.handlers.read().iter() { - if s.have_capability(p) { - to_disconnect.push(p); + if s.have_capability(*p) { + to_disconnect.push(*p); } } } @@ -874,7 +877,7 @@ impl Host { } } for p in to_disconnect { - let h = self.handlers.read().get(p).unwrap().clone(); + let h = self.handlers.read().get(&p).unwrap().clone(); let reserved = self.reserved_nodes.read(); h.disconnected(&NetworkContext::new(io, p, expired_session.clone(), self.sessions.clone(), &reserved), &token); } @@ -980,7 +983,7 @@ impl IoHandler for Host { self.nodes.write().clear_useless(); }, _ => match self.timers.read().get(&token).cloned() { - Some(timer) => match self.handlers.read().get(timer.protocol).cloned() { + Some(timer) => match self.handlers.read().get(&timer.protocol).cloned() { None => { warn!(target: "network", "No handler found for protocol: {:?}", timer.protocol) }, Some(h) => { let reserved = self.reserved_nodes.read(); @@ -1004,11 +1007,11 @@ impl IoHandler for Host { } => { let h = handler.clone(); let reserved = self.reserved_nodes.read(); - h.initialize(&NetworkContext::new(io, protocol, None, self.sessions.clone(), &reserved)); - self.handlers.write().insert(protocol, h); + h.initialize(&NetworkContext::new(io, *protocol, None, self.sessions.clone(), &reserved)); + self.handlers.write().insert(*protocol, h); let mut info = self.info.write(); for v in versions { - info.capabilities.push(CapabilityInfo { protocol: protocol, version: *v, packet_count:0 }); + info.capabilities.push(CapabilityInfo { protocol: *protocol, version: *v, packet_count:0 }); } }, NetworkIoMessage::AddTimer { @@ -1023,7 +1026,7 @@ impl IoHandler for Host { *counter += 1; handler_token }; - self.timers.write().insert(handler_token, ProtocolTimer { protocol: protocol, token: *token }); + self.timers.write().insert(handler_token, ProtocolTimer { protocol: *protocol, token: *token }); io.register_timer(handler_token, *delay).unwrap_or_else(|e| debug!("Error registering timer {}: {:?}", token, e)); }, NetworkIoMessage::Disconnect(ref peer) => { diff --git a/util/network/src/lib.rs b/util/network/src/lib.rs index bfcd49cea..cd0336823 100644 --- a/util/network/src/lib.rs +++ b/util/network/src/lib.rs @@ -45,7 +45,7 @@ //! //! fn main () { //! let mut service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service"); -//! service.register_protocol(Arc::new(MyHandler), "myproto", &[1u8]); +//! service.register_protocol(Arc::new(MyHandler), *b"myp", &[1u8]); //! service.start().expect("Error starting service"); //! //! // Wait for quit condition diff --git a/util/network/src/session.rs b/util/network/src/session.rs index 164248d62..fdba12fff 100644 --- a/util/network/src/session.rs +++ b/util/network/src/session.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::{str, io}; use std::net::SocketAddr; -use std::io; use std::sync::*; use mio::*; use mio::tcp::*; @@ -63,7 +63,7 @@ pub enum SessionData { /// Packet data data: Vec, /// Packet protocol ID - protocol: &'static str, + protocol: [u8; 3], /// Zero based packet ID packet_id: u8, }, @@ -89,15 +89,21 @@ pub struct SessionInfo { #[derive(Debug, PartialEq, Eq)] pub struct PeerCapabilityInfo { - pub protocol: String, + pub protocol: ProtocolId, pub version: u8, } impl Decodable for PeerCapabilityInfo { fn decode(decoder: &D) -> Result where D: Decoder { let c = decoder.as_rlp(); + let p: Vec = try!(c.val_at(0)); + if p.len() != 3 { + return Err(DecoderError::Custom("Invalid subprotocol string length. Should be 3")); + } + let mut p2: ProtocolId = [0u8; 3]; + p2.clone_from_slice(&p); Ok(PeerCapabilityInfo { - protocol: try!(c.val_at(0)), + protocol: p2, version: try!(c.val_at(1)) }) } @@ -105,7 +111,7 @@ impl Decodable for PeerCapabilityInfo { #[derive(Debug)] struct SessionCapabilityInfo { - pub protocol: &'static str, + pub protocol: [u8; 3], pub version: u8, pub packet_count: u8, pub id_offset: u8, @@ -239,12 +245,12 @@ impl Session { } /// Checks if peer supports given capability - pub fn have_capability(&self, protocol: &str) -> bool { + pub fn have_capability(&self, protocol: [u8; 3]) -> bool { self.info.capabilities.iter().any(|c| c.protocol == protocol) } /// Checks if peer supports given capability - pub fn capability_version(&self, protocol: &str) -> Option { + pub fn capability_version(&self, protocol: [u8; 3]) -> Option { self.info.capabilities.iter().filter_map(|c| if c.protocol == protocol { Some(c.version) } else { None }).max() } @@ -270,10 +276,10 @@ impl Session { } /// Send a protocol packet to peer. - pub fn send_packet(&mut self, io: &IoContext, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), NetworkError> + pub fn send_packet(&mut self, io: &IoContext, protocol: [u8; 3], packet_id: u8, data: &[u8]) -> Result<(), NetworkError> where Message: Send + Sync + Clone { if self.info.capabilities.is_empty() || !self.had_hello { - debug!(target: "network", "Sending to unconfirmed session {}, protocol: {}, packet: {}", self.token(), protocol, packet_id); + debug!(target: "network", "Sending to unconfirmed session {}, protocol: {}, packet: {}", self.token(), str::from_utf8(&protocol[..]).unwrap_or("??"), packet_id); return Err(From::from(NetworkError::BadProtocol)); } if self.expired() { diff --git a/util/network/src/tests.rs b/util/network/src/tests.rs index 4186e549a..97a641c81 100644 --- a/util/network/src/tests.rs +++ b/util/network/src/tests.rs @@ -41,7 +41,7 @@ impl TestProtocol { /// Creates and register protocol with the network service pub fn register(service: &mut NetworkService, drop_session: bool) -> Arc { let handler = Arc::new(TestProtocol::new(drop_session)); - service.register_protocol(handler.clone(), "test", &[42u8, 43u8]).expect("Error registering test protocol handler"); + service.register_protocol(handler.clone(), *b"tst", &[42u8, 43u8]).expect("Error registering test protocol handler"); handler } @@ -93,7 +93,7 @@ impl NetworkProtocolHandler for TestProtocol { fn net_service() { let service = NetworkService::new(NetworkConfiguration::new_local()).expect("Error creating network service"); service.start().unwrap(); - service.register_protocol(Arc::new(TestProtocol::new(false)), "myproto", &[1u8]).unwrap(); + service.register_protocol(Arc::new(TestProtocol::new(false)), *b"myp", &[1u8]).unwrap(); } #[test] diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index 708b8d870..df36918dd 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -143,12 +143,12 @@ impl CompactionProfile { } /// Database configuration -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct DatabaseConfig { /// Max number of open files. pub max_open_files: i32, - /// Cache-size - pub cache_size: Option, + /// Cache sizes (in MiB) for specific columns. + pub cache_sizes: HashMap, usize>, /// Compaction profile pub compaction: CompactionProfile, /// Set number of columns @@ -159,17 +159,23 @@ pub struct DatabaseConfig { impl DatabaseConfig { /// Create new `DatabaseConfig` with default parameters and specified set of columns. + /// Note that cache sizes must be explicitly set. pub fn with_columns(columns: Option) -> Self { let mut config = Self::default(); config.columns = columns; config } + + /// Set the column cache size in MiB. + pub fn set_cache(&mut self, col: Option, size: usize) { + self.cache_sizes.insert(col, size); + } } impl Default for DatabaseConfig { fn default() -> DatabaseConfig { DatabaseConfig { - cache_size: None, + cache_sizes: HashMap::new(), max_open_files: 512, compaction: CompactionProfile::default(), columns: None, @@ -213,6 +219,9 @@ impl Database { /// Open database file. Creates if it does not exist. pub fn open(config: &DatabaseConfig, path: &str) -> Result { + // default cache size for columns not specified. + const DEFAULT_CACHE: usize = 2; + let mut opts = Options::new(); if let Some(rate_limit) = config.compaction.write_rate_limit { try!(opts.set_parsed_options(&format!("rate_limiter_bytes_per_sec={}", rate_limit))); @@ -232,17 +241,22 @@ impl Database { let mut cf_options = Vec::with_capacity(config.columns.unwrap_or(0) as usize); - for _ in 0 .. config.columns.unwrap_or(0) { + for col in 0 .. config.columns.unwrap_or(0) { let mut opts = Options::new(); opts.set_compaction_style(DBCompactionStyle::DBUniversalCompaction); opts.set_target_file_size_base(config.compaction.initial_file_size); opts.set_target_file_size_multiplier(config.compaction.file_size_multiplier); - if let Some(cache_size) = config.cache_size { + + let col_opt = config.columns.map(|_| col); + + { + let cache_size = config.cache_sizes.get(&col_opt).cloned().unwrap_or(DEFAULT_CACHE); let mut block_opts = BlockBasedOptions::new(); - // all goes to read cache + // all goes to read cache. block_opts.set_cache(Cache::new(cache_size * 1024 * 1024)); opts.set_block_based_table_factory(&block_opts); } + cf_options.push(opts); } diff --git a/util/src/migration/mod.rs b/util/src/migration/mod.rs index cfd828086..0d34f4198 100644 --- a/util/src/migration/mod.rs +++ b/util/src/migration/mod.rs @@ -20,6 +20,7 @@ mod tests; use std::collections::BTreeMap; use std::fs; +use std::fmt; use std::path::{Path, PathBuf}; use ::kvdb::{CompactionProfile, Database, DatabaseConfig, DBTransaction}; @@ -96,6 +97,17 @@ pub enum Error { Custom(String), } +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match *self { + Error::CannotAddMigration => write!(f, "Cannot add migration"), + Error::MigrationImpossible => write!(f, "Migration impossible"), + Error::Io(ref err) => write!(f, "{}", err), + Error::Custom(ref err) => write!(f, "{}", err), + } + } +} + impl From<::std::io::Error> for Error { fn from(e: ::std::io::Error) -> Self { Error::Io(e) @@ -104,6 +116,8 @@ impl From<::std::io::Error> for Error { /// A generalized migration from the given db to a destination db. pub trait Migration: 'static { + /// Number of columns in the database before the migration. + fn pre_columns(&self) -> Option { self.columns() } /// Number of columns in database after the migration. fn columns(&self) -> Option; /// Version of the database after the migration. @@ -195,6 +209,7 @@ impl Manager { Some(last) => migration.version() > last.version(), None => true, }; + match is_new { true => Ok(self.migrations.push(Box::new(migration))), false => Err(Error::CannotAddMigration), @@ -205,12 +220,14 @@ impl Manager { /// and producing a path where the final migration lives. pub fn execute(&mut self, old_path: &Path, version: u32) -> Result { let config = self.config.clone(); - let columns = self.no_of_columns_at(version); let migrations = self.migrations_from(version); if migrations.is_empty() { return Err(Error::MigrationImpossible) }; + + let columns = migrations.iter().find(|m| m.version() == version).and_then(|m| m.pre_columns()); + let mut db_config = DatabaseConfig { max_open_files: 64, - cache_size: None, + cache_sizes: Default::default(), compaction: config.compaction_profile, columns: columns, wal: true, @@ -267,14 +284,6 @@ impl Manager { fn migrations_from(&mut self, version: u32) -> Vec<&mut Box> { self.migrations.iter_mut().filter(|m| m.version() > version).collect() } - - fn no_of_columns_at(&self, version: u32) -> Option { - let migration = self.migrations.iter().find(|m| m.version() == version); - match migration { - Some(m) => m.columns(), - None => None - } - } } /// Prints a dot every `max` ticks diff --git a/util/src/migration/tests.rs b/util/src/migration/tests.rs index ee5ff574e..05229bee5 100644 --- a/util/src/migration/tests.rs +++ b/util/src/migration/tests.rs @@ -19,7 +19,7 @@ //! are performed in temp sub-directories. use common::*; -use migration::{Config, SimpleMigration, Manager}; +use migration::{Batch, Config, Error, SimpleMigration, Migration, Manager}; use kvdb::Database; use devtools::RandomTempPath; @@ -62,11 +62,10 @@ impl SimpleMigration for Migration0 { fn version(&self) -> u32 { 1 } - fn simple_migrate(&mut self, key: Vec, value: Vec) -> Option<(Vec, Vec)> { - let mut key = key; + fn simple_migrate(&mut self, mut key: Vec, mut value: Vec) -> Option<(Vec, Vec)> { key.push(0x11); - let mut value = value; value.push(0x22); + Some((key, value)) } } @@ -83,6 +82,31 @@ impl SimpleMigration for Migration1 { } } +struct AddsColumn; + +impl Migration for AddsColumn { + fn pre_columns(&self) -> Option { None } + + fn columns(&self) -> Option { Some(1) } + + fn version(&self) -> u32 { 1 } + + fn migrate(&mut self, source: &Database, config: &Config, dest: &mut Database, col: Option) -> Result<(), Error> { + let mut batch = Batch::new(config, col); + + for (key, value) in source.iter(col) { + try!(batch.insert(key.to_vec(), value.to_vec(), dest)); + } + + + if col == Some(1) { + try!(batch.insert(vec![1, 2, 3], vec![4, 5, 6], dest)); + } + + batch.commit(dest) + } +} + #[test] fn one_simple_migration() { let dir = RandomTempPath::create_dir(); @@ -189,3 +213,16 @@ fn is_migration_needed() { assert!(manager.is_needed(1)); assert!(!manager.is_needed(2)); } + +#[test] +fn pre_columns() { + let mut manager = Manager::new(Config::default()); + manager.add_migration(AddsColumn).unwrap(); + + let dir = RandomTempPath::create_dir(); + let db_path = db_path(dir.as_path()); + + // this shouldn't fail to open the database even though it's one column + // short of the one before it. + manager.execute(&db_path, 0).unwrap(); +}