From 5b05cbb128bc63dd33c6a2a0a6a33f4b2529c2b8 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sun, 21 Feb 2016 23:23:46 +0300 Subject: [PATCH 01/48] extended keys with accont meta --- ethtools/Cargo.toml | 10 ++++ ethtools/README.md | 1 + ...--3f49624084b67849c7b4e805c5988c21a430f9d9 | 1 + ...--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf | 1 + ethtools/src/geth_keys.rs | 57 +++++++++++++++++++ ethtools/src/lib.rs | 21 +++++++ util/src/hash.rs | 5 +- util/src/keys/directory.rs | 13 ++++- util/src/keys/store.rs | 12 +++- 9 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 ethtools/Cargo.toml create mode 100644 ethtools/README.md create mode 100644 ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 create mode 100644 ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf create mode 100644 ethtools/src/geth_keys.rs create mode 100644 ethtools/src/lib.rs diff --git a/ethtools/Cargo.toml b/ethtools/Cargo.toml new file mode 100644 index 000000000..5529c1d33 --- /dev/null +++ b/ethtools/Cargo.toml @@ -0,0 +1,10 @@ +[package] +description = "Ethcore Ethereum tools" +homepage = "http://ethcore.io" +license = "GPL-3.0" +name = "ethtools" +version = "0.9.99" +authors = ["Ethcore "] + +[dependencies] +ethcore-util = { path = "../util" } diff --git a/ethtools/README.md b/ethtools/README.md new file mode 100644 index 000000000..f3e852ff5 --- /dev/null +++ b/ethtools/README.md @@ -0,0 +1 @@ +# ethtools diff --git a/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 b/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 new file mode 100644 index 000000000..a62d3056c --- /dev/null +++ b/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 @@ -0,0 +1 @@ +{"address":"3f49624084b67849c7b4e805c5988c21a430f9d9","Crypto":{"cipher":"aes-128-ctr","ciphertext":"9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae","cipherparams":{"iv":"457494bf05f2618c397dc74dbb5181c0"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33"},"mac":"572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e"},"id":"62a0ad73-556d-496a-8e1c-0783d30d3ace","version":3} \ No newline at end of file diff --git a/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf b/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf new file mode 100644 index 000000000..b6caa1c47 --- /dev/null +++ b/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf @@ -0,0 +1 @@ +{"address":"5ba4dcf897e97c2bdf8315b9ef26c13c085988cf","Crypto":{"cipher":"aes-128-ctr","ciphertext":"d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd","cipherparams":{"iv":"89ce5ec129fc27cd5bcbeb8c92bdad50"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a"},"mac":"4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695"},"id":"35086353-fb12-4029-b56b-033cd61ce35b","version":3} \ No newline at end of file diff --git a/ethtools/src/geth_keys.rs b/ethtools/src/geth_keys.rs new file mode 100644 index 000000000..58dc2a0b1 --- /dev/null +++ b/ethtools/src/geth_keys.rs @@ -0,0 +1,57 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Geth keys import/export tool + +use util::hash::*; +use std::path::Path; +use std::result::*; +use std::fs; +use std::str::FromStr; + +/// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)` +pub fn enumerate_geth_keys(path: &Path) -> Result, ::std::io::Error> { + let mut entries = Vec::new(); + for entry in try!(fs::read_dir(path)) { + let entry = try!(entry); + if !try!(fs::metadata(entry.path())).is_dir() { + match entry.file_name().to_str() { + Some(name) => { + let parts: Vec<&str> = name.split("--").collect(); + if parts.len() != 3 { continue; } + match Address::from_str(parts[2]) { + Ok(account_id) => { entries.push((account_id, name.to_owned())); } + Err(e) => { panic!("error: {:?}", e); } + } + }, + None => { continue; } + }; + } + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn can_enumerate() { + let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap(); + assert_eq!(2, keys.len()); + } +} diff --git a/ethtools/src/lib.rs b/ethtools/src/lib.rs new file mode 100644 index 000000000..41dd69cf9 --- /dev/null +++ b/ethtools/src/lib.rs @@ -0,0 +1,21 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Ethereum Tools Library + +extern crate ethcore_util as util; + +pub mod geth_keys; diff --git a/util/src/hash.rs b/util/src/hash.rs index 71c690ef6..a6e8f7950 100644 --- a/util/src/hash.rs +++ b/util/src/hash.rs @@ -235,7 +235,7 @@ macro_rules! impl_hash { } impl serde::Serialize for $from { - fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> + fn serialize(&self, serializer: &mut S) -> Result<(), S::Error> where S: serde::Serializer { let mut hex = "0x".to_owned(); hex.push_str(self.to_hex().as_ref()); @@ -250,7 +250,7 @@ macro_rules! impl_hash { impl serde::de::Visitor for HashVisitor { type Value = $from; - + fn visit_str(&mut self, value: &str) -> Result where E: serde::Error { // 0x + len if value.len() != 2 + $size * 2 { @@ -719,4 +719,3 @@ mod tests { assert_eq!(r, u); } } - diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index bc875db3f..298f67b99 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -333,7 +333,9 @@ pub struct KeyFileContent { /// Holds cypher and decrypt function settings. pub crypto: KeyFileCrypto, /// The identifier. - pub id: Uuid + pub id: Uuid, + /// Account (if present) + pub account: Option
, } #[derive(Debug)] @@ -374,7 +376,8 @@ impl KeyFileContent { KeyFileContent { id: new_uuid(), version: KeyFileVersion::V3(3), - crypto: crypto + crypto: crypto, + account: None } } @@ -407,6 +410,9 @@ impl KeyFileContent { Ok(id) => id }; + let account = as_object.get("account").and_then(|json| json.as_string()).and_then( + |account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None }); + let crypto = match as_object.get("crypto") { None => { return Err(KeyFileParseError::NoCryptoSection); } Some(crypto_json) => match KeyFileCrypto::from_json(crypto_json) { @@ -418,7 +424,8 @@ impl KeyFileContent { Ok(KeyFileContent { version: version, id: id.clone(), - crypto: crypto + crypto: crypto, + account: account }) } diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index ae44d567a..100f3d30c 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -63,12 +63,22 @@ impl SecretStore { /// new instance of Secret Store pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); - path.push(".keys"); + path.push("keystore"); SecretStore { directory: KeyDirectory::new(&path) } } + pub fn accounts(&self) -> Result, ::std::io::Error> { + let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id)) + .filter(|key| key.is_some()) + .map(|key| { let some_key = key.unwrap(); (some_key.account, some_key.id) }) + .filter(|&(ref account, _)| account.is_some()) + .map(|(account, id)| (account.unwrap(), id)) + .collect::>(); + Ok(accounts) + } + #[cfg(test)] fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { From 8bc0b7c77c79ee5ba1868e678c502ba0aa026925 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Sun, 21 Feb 2016 23:44:12 +0300 Subject: [PATCH 02/48] import pub --- util/src/keys/store.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 100f3d30c..284c78b2c 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -69,6 +69,7 @@ impl SecretStore { } } + /// Lists all accounts and corresponding key ids pub fn accounts(&self) -> Result, ::std::io::Error> { let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id)) .filter(|key| key.is_some()) @@ -79,6 +80,21 @@ impl SecretStore { Ok(accounts) } + /// Resolves key_id by account address + pub fn account(&self, account: &Address) -> Option { + let mut accounts = match self.accounts() { + Ok(accounts) => accounts, + Err(e) => { warn!(target: "sstore", "Failed to load accounts: {}", e); return None; } + }; + accounts.retain(|&(ref store_account, _)| account == store_account); + accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone())) + } + + pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> { + try!(self.directory.save(key_file)); + Ok(()) + } + #[cfg(test)] fn new_test(path: &::devtools::RandomTempPath) -> SecretStore { SecretStore { From 753f52fc22b2f5667dd477cc38eac62e1c667eb3 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 22 Feb 2016 20:07:56 +0300 Subject: [PATCH 03/48] geth import finish --- ethtools/Cargo.toml | 2 + ...--3f49624084b67849c7b4e805c5988c21a430f9d9 | 22 +++++++- ...--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf | 22 +++++++- ethtools/src/geth_keys.rs | 50 ++++++++++++++++++- ethtools/src/lib.rs | 2 + 5 files changed, 95 insertions(+), 3 deletions(-) diff --git a/ethtools/Cargo.toml b/ethtools/Cargo.toml index 5529c1d33..0ff934421 100644 --- a/ethtools/Cargo.toml +++ b/ethtools/Cargo.toml @@ -8,3 +8,5 @@ authors = ["Ethcore "] [dependencies] ethcore-util = { path = "../util" } +rustc-serialize = "0.3" +ethcore-devtools = { path = "../devtools" } diff --git a/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 b/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 index a62d3056c..afc376774 100644 --- a/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 +++ b/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 @@ -1 +1,21 @@ -{"address":"3f49624084b67849c7b4e805c5988c21a430f9d9","Crypto":{"cipher":"aes-128-ctr","ciphertext":"9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae","cipherparams":{"iv":"457494bf05f2618c397dc74dbb5181c0"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33"},"mac":"572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e"},"id":"62a0ad73-556d-496a-8e1c-0783d30d3ace","version":3} \ No newline at end of file +{ + "address": "3f49624084b67849c7b4e805c5988c21a430f9d9", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae", + "cipherparams": { + "iv": "457494bf05f2618c397dc74dbb5181c0" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33" + }, + "mac": "572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e" + }, + "id": "62a0ad73-556d-496a-8e1c-0783d30d3ace", + "version": 3 +} diff --git a/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf b/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf index b6caa1c47..b14922037 100644 --- a/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf +++ b/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf @@ -1 +1,21 @@ -{"address":"5ba4dcf897e97c2bdf8315b9ef26c13c085988cf","Crypto":{"cipher":"aes-128-ctr","ciphertext":"d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd","cipherparams":{"iv":"89ce5ec129fc27cd5bcbeb8c92bdad50"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a"},"mac":"4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695"},"id":"35086353-fb12-4029-b56b-033cd61ce35b","version":3} \ No newline at end of file +{ + "address": "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf", + "Crypto": { + "cipher": "aes-128-ctr", + "ciphertext": "d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd", + "cipherparams": { + "iv": "89ce5ec129fc27cd5bcbeb8c92bdad50" + }, + "kdf": "scrypt", + "kdfparams": { + "dklen": 32, + "n": 262144, + "p": 1, + "r": 8, + "salt": "612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a" + }, + "mac": "4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695" + }, + "id": "35086353-fb12-4029-b56b-033cd61ce35b", + "version": 3 +} diff --git a/ethtools/src/geth_keys.rs b/ethtools/src/geth_keys.rs index 58dc2a0b1..3148407dd 100644 --- a/ethtools/src/geth_keys.rs +++ b/ethtools/src/geth_keys.rs @@ -17,13 +17,18 @@ //! Geth keys import/export tool use util::hash::*; +use util::keys::store::SecretStore; +use util::keys::directory::KeyFileContent; use std::path::Path; use std::result::*; use std::fs; use std::str::FromStr; +use std::io; +use std::io::Read; +use rustc_serialize::json::Json; /// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)` -pub fn enumerate_geth_keys(path: &Path) -> Result, ::std::io::Error> { +pub fn enumerate_geth_keys(path: &Path) -> Result, io::Error> { let mut entries = Vec::new(); for entry in try!(fs::read_dir(path)) { let entry = try!(entry); @@ -44,14 +49,57 @@ pub fn enumerate_geth_keys(path: &Path) -> Result, ::std: Ok(entries) } +#[derive(Debug)] +pub enum ImportError { + IoError(io::Error), + FormatError, +} + +impl From for ImportError { + fn from (err: io::Error) -> ImportError { + ImportError::IoError(err) + } +} + +pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> { + let mut file = try!(fs::File::open(geth_keyfile_path)); + let mut buf = String::new(); + try!(file.read_to_string(&mut buf)); + + let mut json = match Json::from_str(&buf) { + Ok(parsed_json) => try!(parsed_json.as_object().ok_or(ImportError::FormatError)).clone(), + Err(_) => { return Err(ImportError::FormatError); } + }; + let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone(); + json.insert("crypto".to_owned(), Json::Object(crypto_object.clone())); + json.remove("Crypto"); + match KeyFileContent::load(&Json::Object(json.clone())) { + Ok(key_file) => try!(secret_store.import_key(key_file)), + Err(_) => { return Err(ImportError::FormatError); } + }; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; use std::path::Path; + use util::hash::*; + use util::keys::store::SecretStore; + use std::str::FromStr; #[test] fn can_enumerate() { let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap(); assert_eq!(2, keys.len()); } + + #[test] + fn can_import() { + let temp = ::devtools::RandomTempPath::new(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap(); + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + } } diff --git a/ethtools/src/lib.rs b/ethtools/src/lib.rs index 41dd69cf9..1509f6d81 100644 --- a/ethtools/src/lib.rs +++ b/ethtools/src/lib.rs @@ -17,5 +17,7 @@ //! Ethereum Tools Library extern crate ethcore_util as util; +extern crate rustc_serialize; +extern crate ethcore_devtools as devtools; pub mod geth_keys; From 61a3e687f5b05b11c92dc3cbf98348ce25d864e9 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 22 Feb 2016 20:08:16 +0300 Subject: [PATCH 04/48] store extra interface --- util/src/keys/directory.rs | 12 +++++++++- util/src/keys/store.rs | 46 +++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 298f67b99..e05ae4877 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -381,6 +381,16 @@ impl KeyFileContent { } } + pub fn load(json: &Json) -> Result { + match Self::from_json(json) { + Ok(key_file) => Ok(key_file), + Err(e) => { + warn!(target: "sstore", "Error parsing json for key: {:?}", e); + Err(()) + } + } + } + /// Returns key file version if it is known. pub fn version(&self) -> Option { match self.version { @@ -410,7 +420,7 @@ impl KeyFileContent { Ok(id) => id }; - let account = as_object.get("account").and_then(|json| json.as_string()).and_then( + let account = as_object.get("address").and_then(|json| json.as_string()).and_then( |account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None }); let crypto = match as_object.get("crypto") { diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 284c78b2c..7aa6d2e76 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -64,8 +64,12 @@ impl SecretStore { pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); path.push("keystore"); + Self::new_in(&path) + } + + pub fn new_in(path: &Path) -> SecretStore { SecretStore { - directory: KeyDirectory::new(&path) + directory: KeyDirectory::new(path) } } @@ -285,6 +289,25 @@ mod tests { result } + fn pregenerate_accounts(temp: &RandomTempPath, count: usize) -> Vec { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + let mut result = Vec::new(); + for i in 0..count { + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(x!(i as u64)); + } + result + } + #[test] fn can_get() { let temp = RandomTempPath::create_dir(); @@ -319,5 +342,26 @@ mod tests { assert_eq!(4, sstore.directory.list().unwrap().len()) } + #[test] + fn can_import_account() { + let temp = RandomTempPath::create_dir(); + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut key_file = + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32)); + key_file.account = Some(Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + let mut sstore = SecretStore::new_test(&temp); + + sstore.import_key(key_file); + + assert_eq!(1, sstore.accounts().unwrap().len()); + assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some()); + } } From a6c41514d423bb9037f9f9b42bbb07cb15328a16 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 22 Feb 2016 18:10:21 +0100 Subject: [PATCH 05/48] Update docs. --- parity/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity/main.rs b/parity/main.rs index f4b7880ab..86d8d11a8 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -74,7 +74,7 @@ Options: --peers NUM Try to manintain that many peers [default: 25]. --no-discovery Disable new peer discovery. --upnp Use UPnP to try to figure out the correct network settings. - --node-key KEY Specify node secret key as hex string. + --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. --cache-max-size BYTES Specify the maximum size of the blockchain cache in bytes [default: 262144]. From 93f2ee66bc2f9561b31abc68adf04fe00f35a7f5 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 22 Feb 2016 22:19:33 +0300 Subject: [PATCH 06/48] geth directory import --- ethtools/Cargo.toml | 1 + ethtools/src/geth_keys.rs | 31 +++++++++++++++++++++++++++++-- ethtools/src/lib.rs | 1 + 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/ethtools/Cargo.toml b/ethtools/Cargo.toml index 0ff934421..87e101b4d 100644 --- a/ethtools/Cargo.toml +++ b/ethtools/Cargo.toml @@ -10,3 +10,4 @@ authors = ["Ethcore "] ethcore-util = { path = "../util" } rustc-serialize = "0.3" ethcore-devtools = { path = "../devtools" } +log = "0.3" diff --git a/ethtools/src/geth_keys.rs b/ethtools/src/geth_keys.rs index 3148407dd..c1a7782d7 100644 --- a/ethtools/src/geth_keys.rs +++ b/ethtools/src/geth_keys.rs @@ -19,7 +19,7 @@ use util::hash::*; use util::keys::store::SecretStore; use util::keys::directory::KeyFileContent; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::result::*; use std::fs; use std::str::FromStr; @@ -61,6 +61,7 @@ impl From for ImportError { } } +/// Imports one geth key to the store pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> { let mut file = try!(fs::File::open(geth_keyfile_path)); let mut buf = String::new(); @@ -80,6 +81,19 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) Ok(()) } +pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { + let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); + for &(ref address, ref file_path) in geth_files.iter() { + let mut path = PathBuf::new(); + path.push(geth_keyfiles_directory); + path.push(file_path); + if let Err(e) = import_geth_key(secret_store, Path::new(&path)) { + warn!("Skipped geth address {}, error importing: {:?}", address, e) + } + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -96,10 +110,23 @@ mod tests { #[test] fn can_import() { - let temp = ::devtools::RandomTempPath::new(); + let temp = ::devtools::RandomTempPath::create_dir(); let mut secret_store = SecretStore::new_in(temp.as_path()); import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap(); let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); assert!(key.is_some()); } + + #[test] + fn can_import_directory() { + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()); + assert!(key.is_some()); + + let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap()); + assert!(key.is_some()); + } } diff --git a/ethtools/src/lib.rs b/ethtools/src/lib.rs index 1509f6d81..4cf365f5f 100644 --- a/ethtools/src/lib.rs +++ b/ethtools/src/lib.rs @@ -19,5 +19,6 @@ extern crate ethcore_util as util; extern crate rustc_serialize; extern crate ethcore_devtools as devtools; +#[macro_use] extern crate log; pub mod geth_keys; From e604c97a435dd763aec9fad8fbceecd65bfe4cf4 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 22 Feb 2016 23:12:13 +0300 Subject: [PATCH 07/48] more account tests --- ethtools/src/geth_keys.rs | 18 ++++++++++++ util/src/keys/directory.rs | 47 ++++++++++++++++++++++++++++-- util/src/keys/store.rs | 59 ++++++++++++++++++++++++++------------ 3 files changed, 103 insertions(+), 21 deletions(-) diff --git a/ethtools/src/geth_keys.rs b/ethtools/src/geth_keys.rs index c1a7782d7..7d6c7adef 100644 --- a/ethtools/src/geth_keys.rs +++ b/ethtools/src/geth_keys.rs @@ -49,9 +49,12 @@ pub fn enumerate_geth_keys(path: &Path) -> Result, io::Er Ok(entries) } +/// Geth import error #[derive(Debug)] pub enum ImportError { + /// Io error reading geth file IoError(io::Error), + /// format error FormatError, } @@ -81,6 +84,7 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) Ok(()) } +/// Imports all geth keys in the directory pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); for &(ref address, ref file_path) in geth_files.iter() { @@ -129,4 +133,18 @@ mod tests { let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap()); assert!(key.is_some()); } + + #[test] + fn can_decrypt_with_imported() { + use util::keys::store::EncryptedHashMap; + use util::bytes::*; + + let temp = ::devtools::RandomTempPath::create_dir(); + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + + let val = secret_store.get::(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123"); + assert!(val.is_ok()); + assert_eq!(vec![0u8, 10], val.unwrap()); + } } diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index e05ae4877..233b8b974 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -381,6 +381,7 @@ impl KeyFileContent { } } + /// Loads key from valid json, returns error and records warning if key is mallformed pub fn load(json: &Json) -> Result { match Self::from_json(json) { Ok(key_file) => Ok(key_file), @@ -444,6 +445,7 @@ impl KeyFileContent { map.insert("id".to_owned(), Json::String(uuid_to_string(&self.id))); map.insert("version".to_owned(), Json::U64(CURRENT_DECLARED_VERSION)); map.insert("crypto".to_owned(), self.crypto.to_json()); + if let Some(ref address) = self.account { map.insert("address".to_owned(), Json::String(format!("{:?}", address))); } Json::Object(map) } } @@ -670,7 +672,7 @@ mod file_tests { } #[test] - fn can_read_scrypt_krf() { + fn can_read_scrypt_kdf() { let json = Json::from_str( r#" { @@ -706,6 +708,47 @@ mod file_tests { } } + #[test] + fn can_read_scrypt_kdf_params() { + let json = Json::from_str( + r#" + { + "crypto" : { + "cipher" : "aes-128-ctr", + "cipherparams" : { + "iv" : "83dbcc02d8ccb40e466191a123791e0e" + }, + "ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c", + "kdf" : "scrypt", + "kdfparams" : { + "dklen" : 32, + "n" : 262144, + "r" : 1, + "p" : 8, + "salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19" + }, + "mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097" + }, + "id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6", + "version" : 3 + } + "#).unwrap(); + + match KeyFileContent::from_json(&json) { + Ok(key_file) => { + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(1, scrypt_params.r); + assert_eq!(8, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + }, + Err(e) => panic!("Error parsing valid file: {:?}", e) + } + } + #[test] fn can_return_error_no_id() { let json = Json::from_str( @@ -861,7 +904,7 @@ mod file_tests { panic!("Should be error of no identifier, got ok"); }, Err(KeyFileParseError::Crypto(CryptoParseError::Scrypt(_))) => { }, - Err(other_error) => { panic!("should be error of no identifier, got {:?}", other_error); } + Err(other_error) => { panic!("should be scrypt parse error, got {:?}", other_error); } } } diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 7aa6d2e76..1fdb1e8a3 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -19,11 +19,12 @@ use keys::directory::*; use common::*; use rcrypto::pbkdf2::*; +use rcrypto::scrypt::*; use rcrypto::hmac::*; use crypto; const KEY_LENGTH: u32 = 32; -const KEY_ITERATIONS: u32 = 4096; +const KEY_ITERATIONS: u32 = 10240; const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize; @@ -60,13 +61,14 @@ pub struct SecretStore { } impl SecretStore { - /// new instance of Secret Store + /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); path.push("keystore"); Self::new_in(&path) } + /// new instance of Secret Store in specific directory pub fn new_in(path: &Path) -> SecretStore { SecretStore { directory: KeyDirectory::new(path) @@ -94,6 +96,7 @@ impl SecretStore { accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone())) } + /// Imports pregenerated key, returns error if not saved correctly pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> { try!(self.directory.save(key_file)); Ok(()) @@ -120,6 +123,15 @@ fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) { derive_key_iterations(password, salt, KEY_ITERATIONS) } +fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) { + let mut derived_key = vec![0u8; KEY_LENGTH_USIZE]; + let scrypt_params = ScryptParams::new(n as u8, r, p); + scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key); + let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE]; + let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE]; + (derived_right_bits.to_vec(), derived_left_bits.to_vec()) +} + fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()]; mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits); @@ -131,24 +143,22 @@ impl EncryptedHashMap for SecretStore { fn get(&self, key: &H128, password: &str) -> Result { match self.directory.get(key) { Some(key_file) => { - let decrypted_bytes = match key_file.crypto.kdf { - KeyFileKdf::Pbkdf2(ref params) => { - let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, ¶ms.salt, params.c); - if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) - .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } - - let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; - match key_file.crypto.cipher_type { - CryptoCipherType::Aes128Ctr(ref iv) => { - crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); - } - } - val - } - _ => { unimplemented!(); } + let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf { + KeyFileKdf::Pbkdf2(ref params) => derive_key_iterations(password, ¶ms.salt, params.c), + KeyFileKdf::Scrypt(ref params) => derive_key_scrypt(password, ¶ms.salt, params.n, params.p, params.r) }; - match Value::from_bytes(&decrypted_bytes) { + if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) + .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } + + let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; + match key_file.crypto.cipher_type { + CryptoCipherType::Aes128Ctr(ref iv) => { + crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); + } + }; + + match Value::from_bytes(&val) { Ok(value) => Ok(value), Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error)) } @@ -304,6 +314,8 @@ mod tests { 262144, 32)); key_file.account = Some(x!(i as u64)); + result.push(key_file.id.clone()); + write_sstore.import_key(key_file).unwrap(); } result } @@ -359,9 +371,18 @@ mod tests { let mut sstore = SecretStore::new_test(&temp); - sstore.import_key(key_file); + sstore.import_key(key_file).unwrap(); assert_eq!(1, sstore.accounts().unwrap().len()); assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some()); } + + #[test] + fn can_list_accounts() { + let temp = RandomTempPath::create_dir(); + pregenerate_accounts(&temp, 30); + let sstore = SecretStore::new_test(&temp); + let accounts = sstore.accounts().unwrap(); + assert_eq!(30, accounts.len()); + } } From 4061799e902901772da2171597df0b5d5b0933d2 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 22 Feb 2016 23:40:38 +0300 Subject: [PATCH 08/48] scrypto decrypt fix --- ethtools/src/geth_keys.rs | 24 +++++++++++++++++++++++- util/src/keys/store.rs | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ethtools/src/geth_keys.rs b/ethtools/src/geth_keys.rs index 7d6c7adef..1a6fb75d4 100644 --- a/ethtools/src/geth_keys.rs +++ b/ethtools/src/geth_keys.rs @@ -134,6 +134,28 @@ mod tests { assert!(key.is_some()); } + #[test] + fn imports_as_scrypt_keys() { + use util::keys::directory::{KeyDirectory, KeyFileKdf}; + let temp = ::devtools::RandomTempPath::create_dir(); + { + let mut secret_store = SecretStore::new_in(temp.as_path()); + import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap(); + } + + let key_directory = KeyDirectory::new(&temp.as_path()); + let key_file = key_directory.get(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap()).unwrap(); + + match key_file.crypto.kdf { + KeyFileKdf::Scrypt(scrypt_params) => { + assert_eq!(262144, scrypt_params.n); + assert_eq!(8, scrypt_params.r); + assert_eq!(1, scrypt_params.p); + }, + _ => { panic!("expected kdf params of crypto to be of scrypt type" ); } + } + } + #[test] fn can_decrypt_with_imported() { use util::keys::store::EncryptedHashMap; @@ -145,6 +167,6 @@ mod tests { let val = secret_store.get::(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123"); assert!(val.is_ok()); - assert_eq!(vec![0u8, 10], val.unwrap()); + assert_eq!(32, val.unwrap().len()); } } diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 1fdb1e8a3..b6ae84990 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -125,7 +125,7 @@ fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) { fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) { let mut derived_key = vec![0u8; KEY_LENGTH_USIZE]; - let scrypt_params = ScryptParams::new(n as u8, r, p); + let scrypt_params = ScryptParams::new(n.trailing_zeros() as u8, r, p); scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key); let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE]; let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE]; From bceafe9094eaf21a4a233b76fb9d3df0a6c638f5 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Tue, 23 Feb 2016 00:05:28 +0300 Subject: [PATCH 09/48] fix import statement --- util/src/keys/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index b6ae84990..3c2aef7c3 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -356,8 +356,8 @@ mod tests { #[test] fn can_import_account() { - let temp = RandomTempPath::create_dir(); use keys::directory::{KeyFileContent, KeyFileCrypto}; + let temp = RandomTempPath::create_dir(); let mut key_file = KeyFileContent::new( KeyFileCrypto::new_pbkdf2( From 990c5c8faa6cded59257846bd58039e0c48ad085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 23 Feb 2016 18:44:13 +0100 Subject: [PATCH 10/48] Refactoring client and fixing mark_as_bad & SyncMessage bugs --- ethcore/src/client.rs | 145 +++++++++++++++++++++++++---------------- ethcore/src/service.rs | 7 +- 2 files changed, 95 insertions(+), 57 deletions(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index c3ec4b4d0..aa71fbbfa 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -22,7 +22,7 @@ use rocksdb::{Options, DB, DBCompactionStyle}; use blockchain::{BlockChain, BlockProvider, CacheSize}; use views::BlockView; use error::*; -use header::BlockNumber; +use header::{BlockNumber, Header}; use state::State; use spec::Spec; use engine::Engine; @@ -243,85 +243,117 @@ impl Client { self.block_queue.write().unwrap().flush(); } + fn build_last_hashes(&self, header: &Header) -> LastHashes { + let mut last_hashes = LastHashes::new(); + last_hashes.resize(256, H256::new()); + last_hashes[0] = header.parent_hash.clone(); + let chain = self.chain.read().unwrap(); + for i in 0..255 { + match chain.block_details(&last_hashes[i]) { + Some(details) => { + last_hashes[i + 1] = details.parent.clone(); + }, + None => break, + } + } + last_hashes + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, io: &IoChannel) -> usize { - let mut ret = 0; - let mut bad = HashSet::new(); + let max_blocks_to_import = 128; + + let mut imported = 0; + let mut good_blocks = Vec::with_capacity(max_blocks_to_import); + let mut bad_blocks = HashSet::new(); + let engine = self.engine.deref().deref(); + let _import_lock = self.import_lock.lock(); - let blocks = self.block_queue.write().unwrap().drain(128); - let mut good_blocks = Vec::with_capacity(128); + let blocks = self.block_queue.write().unwrap().drain(max_blocks_to_import); + for block in blocks { - if bad.contains(&block.header.parent_hash) { - self.block_queue.write().unwrap().mark_as_bad(&block.header.hash()); - bad.insert(block.header.hash()); + let header = &block.header; + let header_hash = block.header.hash(); + let bad_contains_parent = bad_blocks.contains(&header.parent_hash); + + let mark_block_as_bad = || { + self.block_queue.write().unwrap().mark_as_bad(&header_hash); + bad_blocks.insert(header_hash); + }; + + if bad_contains_parent { + mark_block_as_bad(); continue; } - let header = &block.header; - if let Err(e) = verify_block_family(&header, &block.bytes, self.engine.deref().deref(), self.chain.read().unwrap().deref()) { + // Verify Block Family + let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); + if let Err(e) = verify_family_result { warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); + mark_block_as_bad(); break; }; - let parent = match self.chain.read().unwrap().block_header(&header.parent_hash) { - Some(p) => p, - None => { - warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - bad.insert(block.header.hash()); - break; - }, - }; - // build last hashes - let mut last_hashes = LastHashes::new(); - last_hashes.resize(256, H256::new()); - last_hashes[0] = header.parent_hash.clone(); - for i in 0..255 { - match self.chain.read().unwrap().block_details(&last_hashes[i]) { - Some(details) => { - last_hashes[i + 1] = details.parent.clone(); - }, - None => break, - } - } + // Check if Parent is in chain + let chain_has_parent = self.chain.read().unwrap().block_header(&header.parent_hash); + if let None = chain_has_parent { + warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + mark_block_as_bad(); + break; + }; + + // Enact Verified Block + let parent = chain_has_parent.unwrap(); + let last_hashes = self.build_last_hashes(header); let db = self.state_db.lock().unwrap().clone(); - let result = match enact_verified(&block, self.engine.deref().deref(), db, &parent, &last_hashes) { - Ok(b) => b, - Err(e) => { - warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - bad.insert(block.header.hash()); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); - break; - } + + let enact_result = enact_verified(&block, engine, db, &parent, &last_hashes); + if let Err(e) = enact_result { + warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + mark_block_as_bad(); + break; }; - if let Err(e) = verify_block_final(&header, result.block().header()) { + + // Final Verification + let enact_result = enact_result.unwrap(); + if let Err(e) = verify_block_final(&header, enact_result.block().header()) { warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - self.block_queue.write().unwrap().mark_as_bad(&header.hash()); + mark_block_as_bad(); break; } - good_blocks.push(header.hash().clone()); - + // Insert block self.chain.write().unwrap().insert_block(&block.bytes); //TODO: err here? - let ancient = if header.number() >= HISTORY { Some(header.number() - HISTORY) } else { None }; - match result.drain().commit(header.number(), &header.hash(), ancient.map(|n|(n, self.chain.read().unwrap().block_hash(n).unwrap()))) { - Ok(_) => (), - Err(e) => { - warn!(target: "client", "State DB commit failed: {:?}", e); - break; - } + good_blocks.push(header.hash()); + + let ancient = if header.number() >= HISTORY { + let n = header.number() - HISTORY; + let chain = self.chain.read().unwrap(); + Some((n, chain.block_hash(n).unwrap())) + } else { + None + }; + + // Commit results + let commit_result = enact_result.drain().commit(header.number(), &header.hash(), ancient); + if let Err(e) = commit_result { + warn!(target: "client", "State DB commit failed: {:?}", e); + break; } + self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); - ret += 1; + imported += 1; } + self.block_queue.write().unwrap().mark_as_good(&good_blocks); if !good_blocks.is_empty() && self.block_queue.read().unwrap().queue_info().is_empty() { - io.send(NetworkIoMessage::User(SyncMessage::BlockVerified)).unwrap(); + io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { + good: good_blocks, + bad: bad_blocks.into_iter().collect(), + })).unwrap(); } - ret + imported } /// Get a copy of the best block's state. @@ -393,7 +425,7 @@ impl BlockChainClient for Client { None => BlockStatus::Unknown } } - + fn block_total_difficulty(&self, id: BlockId) -> Option { let chain = self.chain.read().unwrap(); Self::block_hash(&chain, id).and_then(|hash| chain.block_details(&hash)).map(|d| d.total_difficulty) @@ -435,6 +467,7 @@ impl BlockChainClient for Client { return Err(ImportError::UnknownParent); } self.block_queue.write().unwrap().import_block(bytes) + // TODO [ToDr] remove transactions } fn queue_info(&self) -> BlockQueueInfo { diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 534aab49d..cdf3425e8 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -26,7 +26,12 @@ use client::Client; #[derive(Clone)] pub enum SyncMessage { /// New block has been imported into the blockchain - NewChainBlock(Bytes), //TODO: use Cow + NewChainBlocks { + /// Hashes of blocks imported to blockchain + good: Vec, + /// Hashes of blocks not imported to blockchain + bad: Vec, + }, /// A block is ready BlockVerified, } From 4084acd869f53e0695bed887bcf33578b611c681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 23 Feb 2016 18:51:18 +0100 Subject: [PATCH 11/48] Removing dangling comment --- ethcore/src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index aa71fbbfa..87dba3dd5 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -467,7 +467,6 @@ impl BlockChainClient for Client { return Err(ImportError::UnknownParent); } self.block_queue.write().unwrap().import_block(bytes) - // TODO [ToDr] remove transactions } fn queue_info(&self) -> BlockQueueInfo { From 8beba717f8456fa205471a02eb3472a41fa320fd Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 23 Feb 2016 19:38:06 +0100 Subject: [PATCH 12/48] Delayed UPnP initialization --- parity/main.rs | 6 +- util/src/network/host.rs | 136 +++++++++++++++++++++--------------- util/src/network/service.rs | 1 - 3 files changed, 84 insertions(+), 59 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index e25933f2d..79ee41590 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -73,7 +73,7 @@ Options: --address URL Equivalent to --listen-address URL --public-address URL. --peers NUM Try to manintain that many peers [default: 25]. --no-discovery Disable new peer discovery. - --upnp Use UPnP to try to figure out the correct network settings. + --no-upnp Disable trying to figure out the correct public adderss over UPnP. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. --cache-pref-size BYTES Specify the prefered size of the blockchain cache in bytes [default: 16384]. @@ -101,7 +101,7 @@ struct Args { flag_address: Option, flag_peers: u32, flag_no_discovery: bool, - flag_upnp: bool, + flag_no_upnp: bool, flag_node_key: Option, flag_cache_pref_size: usize, flag_cache_max_size: usize, @@ -233,7 +233,7 @@ impl Configuration { fn net_settings(&self, spec: &Spec) -> NetworkConfiguration { let mut ret = NetworkConfiguration::new(); - ret.nat_enabled = self.args.flag_upnp; + ret.nat_enabled = !self.args.flag_no_upnp; ret.boot_nodes = self.init_nodes(spec); let (listen, public) = self.net_addresses(); ret.listen_address = listen; diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 8dd9eb9cc..feb342700 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -106,6 +106,7 @@ const IDLE: usize = LAST_HANDSHAKE + 2; const DISCOVERY: usize = LAST_HANDSHAKE + 3; const DISCOVERY_REFRESH: usize = LAST_HANDSHAKE + 4; const DISCOVERY_ROUND: usize = LAST_HANDSHAKE + 5; +const INIT_PUBLIC: usize = LAST_HANDSHAKE + 6; const FIRST_SESSION: usize = 0; const LAST_SESSION: usize = FIRST_SESSION + MAX_SESSIONS - 1; const FIRST_HANDSHAKE: usize = LAST_SESSION + 1; @@ -261,7 +262,9 @@ pub struct HostInfo { /// TCP connection port. pub listen_port: u16, /// Registered capabilities (handlers) - pub capabilities: Vec + pub capabilities: Vec, + /// Public address + discovery port + public_endpoint: NodeEndpoint, } impl HostInfo { @@ -294,16 +297,15 @@ struct ProtocolTimer { /// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host where Message: Send + Sync + Clone { pub info: RwLock, - tcp_listener: Mutex, + tcp_listener: Mutex>, handshakes: Arc>>, sessions: Arc>>, - discovery: Option>, + discovery: Mutex>, nodes: RwLock, handlers: RwLock>>>, timers: RwLock>, timer_counter: RwLock, stats: Arc, - public_endpoint: NodeEndpoint, pinned_nodes: Vec, } @@ -316,27 +318,6 @@ impl Host where Message: Send + Sync + Clone { }; let udp_port = config.udp_port.unwrap_or(listen_address.port()); - let public_endpoint = match config.public_address { - None => { - let public_address = select_public_address(listen_address.port()); - let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; - if config.nat_enabled { - match map_external_address(&local_endpoint) { - Some(endpoint) => { - info!("NAT Mappped to external address {}", endpoint.address); - endpoint - }, - None => local_endpoint - } - } else { - local_endpoint - } - } - Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } - }; - - // Setup the server socket - let tcp_listener = TcpListener::bind(&listen_address).unwrap(); let keys = if let Some(ref secret) = config.use_secret { KeyPair::from_secret(secret.clone()).unwrap() } else { @@ -350,10 +331,8 @@ impl Host where Message: Send + Sync + Clone { }, |s| KeyPair::from_secret(s).expect("Error creating node secret key")) }; - let discovery = if config.discovery_enabled && !config.pin { - Some(Discovery::new(&keys, listen_address.clone(), public_endpoint.clone(), DISCOVERY)) - } else { None }; let path = config.config_path.clone(); + let local_endpoint = NodeEndpoint { address: listen_address, udp_port: udp_port }; let mut host = Host:: { info: RwLock::new(HostInfo { keys: keys, @@ -363,9 +342,10 @@ impl Host where Message: Send + Sync + Clone { client_version: version(), listen_port: 0, capabilities: Vec::new(), + public_endpoint: local_endpoint, // will be replaced by public once it is resolved }), - discovery: discovery.map(Mutex::new), - tcp_listener: Mutex::new(tcp_listener), + discovery: Mutex::new(None), + tcp_listener: Mutex::new(None), handshakes: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_HANDSHAKE, MAX_HANDSHAKES))), sessions: Arc::new(RwLock::new(Slab::new_starting_at(FIRST_SESSION, MAX_SESSIONS))), nodes: RwLock::new(NodeTable::new(path)), @@ -373,16 +353,12 @@ impl Host where Message: Send + Sync + Clone { timers: RwLock::new(HashMap::new()), timer_counter: RwLock::new(USER_TIMER), stats: Arc::new(NetworkStats::default()), - public_endpoint: public_endpoint, pinned_nodes: Vec::new(), }; let port = listen_address.port(); host.info.write().unwrap().deref_mut().listen_port = port; let boot_nodes = host.info.read().unwrap().config.boot_nodes.clone(); - if let Some(ref mut discovery) = host.discovery { - discovery.lock().unwrap().init_node_list(host.nodes.read().unwrap().unordered_entries()); - } for n in boot_nodes { host.add_node(&n); } @@ -400,8 +376,8 @@ impl Host where Message: Send + Sync + Clone { let entry = NodeEntry { endpoint: n.endpoint.clone(), id: n.id.clone() }; self.pinned_nodes.push(n.id.clone()); self.nodes.write().unwrap().add_node(n); - if let Some(ref mut discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + if let &mut Some(ref mut discovery) = self.discovery.lock().unwrap().deref_mut() { + discovery.add_node(entry); } } } @@ -412,7 +388,61 @@ impl Host where Message: Send + Sync + Clone { } pub fn client_url(&self) -> String { - format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.public_endpoint.clone())) + format!("{}", Node::new(self.info.read().unwrap().id().clone(), self.info.read().unwrap().public_endpoint.clone())) + } + + fn init_public_interface(&self, io: &IoContext>) { + io.clear_timer(INIT_PUBLIC).unwrap(); + let mut tcp_listener = self.tcp_listener.lock().unwrap(); + if tcp_listener.is_some() { + return; + } + // public_endpoint in host info contains local adderss at this point + let listen_address = self.info.read().unwrap().public_endpoint.address.clone(); + let udp_port = self.info.read().unwrap().config.udp_port.unwrap_or(listen_address.port()); + let public_endpoint = match self.info.read().unwrap().config.public_address { + None => { + let public_address = select_public_address(listen_address.port()); + let local_endpoint = NodeEndpoint { address: public_address, udp_port: udp_port }; + if self.info.read().unwrap().config.nat_enabled { + match map_external_address(&local_endpoint) { + Some(endpoint) => { + info!("NAT mappped to external address {}", endpoint.address); + endpoint + }, + None => local_endpoint + } + } else { + local_endpoint + } + } + Some(addr) => NodeEndpoint { address: addr, udp_port: udp_port } + }; + + // Setup the server socket + *tcp_listener = Some(TcpListener::bind(&listen_address).unwrap()); + self.info.write().unwrap().public_endpoint = public_endpoint.clone(); + io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); + info!("Public node URL: {}", self.client_url()); + + // Initialize discovery. + let discovery = { + let info = self.info.read().unwrap(); + if info.config.discovery_enabled && !info.config.pin { + Some(Discovery::new(&info.keys, listen_address.clone(), public_endpoint, DISCOVERY)) + } else { None } + }; + + if let Some(mut discovery) = discovery { + discovery.init_node_list(self.nodes.read().unwrap().unordered_entries()); + for n in self.nodes.read().unwrap().unordered_entries() { + discovery.add_node(n.clone()); + } + io.register_stream(DISCOVERY).expect("Error registering UDP listener"); + io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); + io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); + *self.discovery.lock().unwrap().deref_mut() = Some(discovery); + } } fn maintain_network(&self, io: &IoContext>) { @@ -526,7 +556,7 @@ impl Host where Message: Send + Sync + Clone { fn accept(&self, io: &IoContext>) { trace!(target: "network", "Accepting incoming connection"); loop { - let socket = match self.tcp_listener.lock().unwrap().accept() { + let socket = match self.tcp_listener.lock().unwrap().as_ref().unwrap().accept() { Ok(None) => break, Ok(Some((sock, _addr))) => sock, Err(e) => { @@ -666,8 +696,9 @@ impl Host where Message: Send + Sync + Clone { if let Ok(address) = session.remote_addr() { let entry = NodeEntry { id: session.id().clone(), endpoint: NodeEndpoint { address: address, udp_port: address.port() } }; self.nodes.write().unwrap().add_node(Node::new(entry.id.clone(), entry.endpoint.clone())); - if let Some(ref discovery) = self.discovery { - discovery.lock().unwrap().add_node(entry); + let mut discovery = self.discovery.lock().unwrap(); + if let &mut Some(ref mut discovery) = discovery.deref_mut() { + discovery.add_node(entry); } } } @@ -760,13 +791,8 @@ impl Host where Message: Send + Sync + Clone { impl IoHandler> for Host where Message: Send + Sync + Clone + 'static { /// Initialize networking fn initialize(&self, io: &IoContext>) { - io.register_stream(TCP_ACCEPT).expect("Error registering TCP listener"); io.register_timer(IDLE, MAINTENANCE_TIMEOUT).expect("Error registering Network idle timer"); - if self.discovery.is_some() { - io.register_stream(DISCOVERY).expect("Error registering UDP listener"); - io.register_timer(DISCOVERY_REFRESH, 7200).expect("Error registering discovery timer"); - io.register_timer(DISCOVERY_ROUND, 300).expect("Error registering discovery timer"); - } + io.register_timer(INIT_PUBLIC, 0).expect("Error registering initialization timer"); self.maintain_network(io) } @@ -784,7 +810,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_readable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(stream, io), DISCOVERY => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().readable() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().readable() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -800,7 +826,7 @@ impl IoHandler> for Host where Messa FIRST_SESSION ... LAST_SESSION => self.session_writable(stream, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(stream, io), DISCOVERY => { - self.discovery.as_ref().unwrap().lock().unwrap().writable(); + self.discovery.lock().unwrap().as_mut().unwrap().writable(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); } _ => panic!("Received unknown writable token"), @@ -810,14 +836,15 @@ impl IoHandler> for Host where Messa fn timeout(&self, io: &IoContext>, token: TimerToken) { match token { IDLE => self.maintain_network(io), + INIT_PUBLIC => self.init_public_interface(io), FIRST_SESSION ... LAST_SESSION => self.connection_timeout(token, io), FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.connection_timeout(token, io), DISCOVERY_REFRESH => { - self.discovery.as_ref().unwrap().lock().unwrap().refresh(); + self.discovery.lock().unwrap().as_mut().unwrap().refresh(); io.update_registration(DISCOVERY).expect("Error updating discovery registration"); }, DISCOVERY_ROUND => { - let node_changes = { self.discovery.as_ref().unwrap().lock().unwrap().round() }; + let node_changes = { self.discovery.lock().unwrap().as_mut().unwrap().round() }; if let Some(node_changes) = node_changes { self.update_nodes(io, node_changes); } @@ -892,8 +919,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().register_socket(reg, event_loop).expect("Error registering socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), - TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().register_socket(event_loop).expect("Error registering discovery socket"), + TCP_ACCEPT => event_loop.register(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error registering stream"), _ => warn!("Unexpected stream registration") } } @@ -915,7 +942,6 @@ impl IoHandler> for Host where Messa } } DISCOVERY => (), - TCP_ACCEPT => event_loop.deregister(self.tcp_listener.lock().unwrap().deref()).unwrap(), _ => warn!("Unexpected stream deregistration") } } @@ -934,8 +960,8 @@ impl IoHandler> for Host where Messa connection.lock().unwrap().update_socket(reg, event_loop).expect("Error updating socket"); } } - DISCOVERY => self.discovery.as_ref().unwrap().lock().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), - TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().deref(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), + DISCOVERY => self.discovery.lock().unwrap().as_ref().unwrap().update_registration(event_loop).expect("Error reregistering discovery socket"), + TCP_ACCEPT => event_loop.reregister(self.tcp_listener.lock().unwrap().as_ref().unwrap(), Token(TCP_ACCEPT), EventSet::all(), PollOpt::edge()).expect("Error reregistering stream"), _ => warn!("Unexpected stream update") } } diff --git a/util/src/network/service.rs b/util/src/network/service.rs index 1cd48abe1..7b9388e85 100644 --- a/util/src/network/service.rs +++ b/util/src/network/service.rs @@ -42,7 +42,6 @@ impl NetworkService where Message: Send + Sync + Clone + 'stat let host = Arc::new(Host::new(config)); let stats = host.stats().clone(); let host_info = host.client_version(); - info!("Node URL: {}", host.client_url()); try!(io_service.register_handler(host)); Ok(NetworkService { io_service: io_service, From 5bd355e0af9baf2b831c8ebc19e166794b42d5c6 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 24 Feb 2016 09:55:09 +0300 Subject: [PATCH 13/48] removing extra crate --- ethtools/Cargo.toml | 13 ---------- ethtools/README.md | 1 - ethtools/src/lib.rs | 24 ----------------- ...--3f49624084b67849c7b4e805c5988c21a430f9d9 | 0 ...--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf | 0 util/src/keys/directory.rs | 2 ++ .../src/keys/geth_import.rs | 26 +++++++------------ util/src/keys/mod.rs | 1 + util/src/keys/store.rs | 14 ++++++++++ 9 files changed, 26 insertions(+), 55 deletions(-) delete mode 100644 ethtools/Cargo.toml delete mode 100644 ethtools/README.md delete mode 100644 ethtools/src/lib.rs rename {ethtools => util}/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 (100%) rename {ethtools => util}/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf (100%) rename ethtools/src/geth_keys.rs => util/src/keys/geth_import.rs (91%) diff --git a/ethtools/Cargo.toml b/ethtools/Cargo.toml deleted file mode 100644 index 87e101b4d..000000000 --- a/ethtools/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -description = "Ethcore Ethereum tools" -homepage = "http://ethcore.io" -license = "GPL-3.0" -name = "ethtools" -version = "0.9.99" -authors = ["Ethcore "] - -[dependencies] -ethcore-util = { path = "../util" } -rustc-serialize = "0.3" -ethcore-devtools = { path = "../devtools" } -log = "0.3" diff --git a/ethtools/README.md b/ethtools/README.md deleted file mode 100644 index f3e852ff5..000000000 --- a/ethtools/README.md +++ /dev/null @@ -1 +0,0 @@ -# ethtools diff --git a/ethtools/src/lib.rs b/ethtools/src/lib.rs deleted file mode 100644 index 4cf365f5f..000000000 --- a/ethtools/src/lib.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015, 2016 Ethcore (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Ethereum Tools Library - -extern crate ethcore_util as util; -extern crate rustc_serialize; -extern crate ethcore_devtools as devtools; -#[macro_use] extern crate log; - -pub mod geth_keys; diff --git a/ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 b/util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 similarity index 100% rename from ethtools/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 rename to util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9 diff --git a/ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf b/util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf similarity index 100% rename from ethtools/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf rename to util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 233b8b974..7fe6fb1ee 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -618,6 +618,8 @@ impl KeyDirectory { Err(_) => Err(KeyFileLoadError::ParseError(KeyFileParseError::InvalidJson)) } } + + } diff --git a/ethtools/src/geth_keys.rs b/util/src/keys/geth_import.rs similarity index 91% rename from ethtools/src/geth_keys.rs rename to util/src/keys/geth_import.rs index 1a6fb75d4..61df88589 100644 --- a/ethtools/src/geth_keys.rs +++ b/util/src/keys/geth_import.rs @@ -16,16 +16,9 @@ //! Geth keys import/export tool -use util::hash::*; -use util::keys::store::SecretStore; -use util::keys::directory::KeyFileContent; -use std::path::{Path, PathBuf}; -use std::result::*; -use std::fs; -use std::str::FromStr; -use std::io; -use std::io::Read; -use rustc_serialize::json::Json; +use common::*; +use keys::store::SecretStore; +use keys::directory::KeyFileContent; /// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)` pub fn enumerate_geth_keys(path: &Path) -> Result, io::Error> { @@ -86,6 +79,7 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) /// Imports all geth keys in the directory pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> { + use std::path::PathBuf; let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory)); for &(ref address, ref file_path) in geth_files.iter() { let mut path = PathBuf::new(); @@ -101,10 +95,9 @@ pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: #[cfg(test)] mod tests { use super::*; - use std::path::Path; - use util::hash::*; - use util::keys::store::SecretStore; - use std::str::FromStr; + use common::*; + use keys::store::SecretStore; + #[test] fn can_enumerate() { @@ -136,7 +129,7 @@ mod tests { #[test] fn imports_as_scrypt_keys() { - use util::keys::directory::{KeyDirectory, KeyFileKdf}; + use keys::directory::{KeyDirectory, KeyFileKdf}; let temp = ::devtools::RandomTempPath::create_dir(); { let mut secret_store = SecretStore::new_in(temp.as_path()); @@ -158,8 +151,7 @@ mod tests { #[test] fn can_decrypt_with_imported() { - use util::keys::store::EncryptedHashMap; - use util::bytes::*; + use keys::store::EncryptedHashMap; let temp = ::devtools::RandomTempPath::create_dir(); let mut secret_store = SecretStore::new_in(temp.as_path()); diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index fd52136d7..b9c9dcb08 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -18,3 +18,4 @@ pub mod directory; pub mod store; +mod geth_import; diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 3c2aef7c3..3b1ca4727 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -75,6 +75,20 @@ impl SecretStore { } } + /// trys to import keys in the known locations + pub fn try_import_existing(&mut self) { + use std::path::PathBuf; + use keys::geth_import; + + let mut import_path = PathBuf::new(); + import_path.push(::std::env::home_dir().expect("Failed to get home dir")); + import_path.push(".ethereum"); + import_path.push("keystore"); + if let Err(e) = geth_import::import_geth_keys(self, &import_path) { + warn!(target: "sstore", "Error retrieving geth keys: {:?}", e) + } + } + /// Lists all accounts and corresponding key ids pub fn accounts(&self) -> Result, ::std::io::Error> { let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id)) From d3fe3f26918c676353f96461a94ac6a8aa6fceb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 24 Feb 2016 10:55:34 +0100 Subject: [PATCH 14/48] Client refactoring [WIP] --- ethcore/src/block_queue.rs | 23 ++++--- ethcore/src/client.rs | 124 ++++++++++++++++++++----------------- 2 files changed, 81 insertions(+), 66 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index c39f158f0..b802e3c26 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -285,19 +285,22 @@ impl BlockQueue { } /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&mut self, hash: &H256) { + pub fn mark_as_bad(&mut self, hashes: &[H256]) { let mut verification_lock = self.verification.lock().unwrap(); let mut verification = verification_lock.deref_mut(); - verification.bad.insert(hash.clone()); - self.processing.write().unwrap().remove(&hash); let mut new_verified = VecDeque::new(); - for block in verification.verified.drain(..) { - if verification.bad.contains(&block.header.parent_hash) { - verification.bad.insert(block.header.hash()); - self.processing.write().unwrap().remove(&block.header.hash()); - } - else { - new_verified.push_back(block); + + for hash in hashes { + verification.bad.insert(hash.clone()); + self.processing.write().unwrap().remove(&hash); + for block in verification.verified.drain(..) { + if verification.bad.contains(&block.header.parent_hash) { + verification.bad.insert(block.header.hash()); + self.processing.write().unwrap().remove(&block.header.hash()); + } + else { + new_verified.push_back(block); + } } } verification.verified = new_verified; diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 87dba3dd5..3c44b97bf 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -259,70 +259,73 @@ impl Client { last_hashes } + fn check_and_close_block(&self, block: &PreVerifiedBlock) -> Result { + let engine = self.engine.deref().deref(); + let header = &block.header; + let header_hash = block.header.hash(); + + // Verify Block Family + let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); + if let Err(e) = verify_family_result { + warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Check if Parent is in chain + let chain_has_parent = self.chain.read().unwrap().block_header(&header.parent_hash); + if let None = chain_has_parent { + warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); + return Err(()); + }; + + // Enact Verified Block + let parent = chain_has_parent.unwrap(); + let last_hashes = self.build_last_hashes(header); + let db = self.state_db.lock().unwrap().clone(); + + let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); + if let Err(e) = enact_result { + warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + }; + + // Final Verification + let closed_block = enact_result.unwrap(); + if let Err(e) = verify_block_final(&header, closed_block.block().header()) { + warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); + return Err(()); + } + + Ok(closed_block) + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, io: &IoChannel) -> usize { let max_blocks_to_import = 128; - let mut imported = 0; let mut good_blocks = Vec::with_capacity(max_blocks_to_import); let mut bad_blocks = HashSet::new(); - let engine = self.engine.deref().deref(); let _import_lock = self.import_lock.lock(); let blocks = self.block_queue.write().unwrap().drain(max_blocks_to_import); for block in blocks { - let header = &block.header; - let header_hash = block.header.hash(); - let bad_contains_parent = bad_blocks.contains(&header.parent_hash); + let header = block.header; - let mark_block_as_bad = || { - self.block_queue.write().unwrap().mark_as_bad(&header_hash); - bad_blocks.insert(header_hash); - }; - - if bad_contains_parent { - mark_block_as_bad(); + if bad_blocks.contains(&header.parent_hash) { + bad_blocks.insert(header.hash()); continue; } - // Verify Block Family - let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); - if let Err(e) = verify_family_result { - warn!(target: "client", "Stage 3 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - mark_block_as_bad(); - break; - }; - // Check if Parent is in chain - let chain_has_parent = self.chain.read().unwrap().block_header(&header.parent_hash); - if let None = chain_has_parent { - warn!(target: "client", "Block import failed for #{} ({}): Parent not found ({}) ", header.number(), header.hash(), header.parent_hash); - mark_block_as_bad(); - break; - }; - - // Enact Verified Block - let parent = chain_has_parent.unwrap(); - let last_hashes = self.build_last_hashes(header); - let db = self.state_db.lock().unwrap().clone(); - - let enact_result = enact_verified(&block, engine, db, &parent, &last_hashes); - if let Err(e) = enact_result { - warn!(target: "client", "Block import failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - mark_block_as_bad(); - break; - }; - - // Final Verification - let enact_result = enact_result.unwrap(); - if let Err(e) = verify_block_final(&header, enact_result.block().header()) { - warn!(target: "client", "Stage 4 block verification failed for #{} ({})\nError: {:?}", header.number(), header.hash(), e); - mark_block_as_bad(); + let closed_block = self.check_and_close_block(&block); + if let Err(_) = closed_block { + bad_blocks.insert(header.hash()); break; } // Insert block + let closed_block = closed_block.unwrap(); self.chain.write().unwrap().insert_block(&block.bytes); //TODO: err here? good_blocks.push(header.hash()); @@ -335,24 +338,33 @@ impl Client { }; // Commit results - let commit_result = enact_result.drain().commit(header.number(), &header.hash(), ancient); - if let Err(e) = commit_result { - warn!(target: "client", "State DB commit failed: {:?}", e); - break; - } + closed_block.drain() + .commit(header.number(), &header.hash(), ancient) + .expect("State DB commit failed."); self.report.write().unwrap().accrue_block(&block); trace!(target: "client", "Imported #{} ({})", header.number(), header.hash()); - imported += 1; } - self.block_queue.write().unwrap().mark_as_good(&good_blocks); - if !good_blocks.is_empty() && self.block_queue.read().unwrap().queue_info().is_empty() { - io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { - good: good_blocks, - bad: bad_blocks.into_iter().collect(), - })).unwrap(); + let imported = good_blocks.len(); + let bad_blocks = bad_blocks.into_iter().collect::>(); + + { + let block_queue = self.block_queue.write().unwrap(); + block_queue.mark_as_bad(&bad_blocks); + block_queue.mark_as_good(&good_blocks); } + + { + let block_queue = self.block_queue.read().unwrap(); + if !good_blocks.is_empty() && block_queue.queue_info().is_empty() { + io.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { + good: good_blocks, + bad: bad_blocks, + })).unwrap(); + } + } + imported } From d914a27bdfd2ac69035f129600ee719d7337794d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 24 Feb 2016 11:17:25 +0100 Subject: [PATCH 15/48] Removing lifetimes from Blocks --- ethcore/src/block.rs | 38 +++++++++++++++++----------------- ethcore/src/client.rs | 6 ++---- ethcore/src/ethereum/ethash.rs | 4 ++-- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index a1194c665..9051b6d9a 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -146,18 +146,18 @@ impl IsBlock for ExecutedBlock { /// /// It's a bit like a Vec, eccept that whenever a transaction is pushed, we execute it and /// maintain the system `state()`. We also archive execution receipts in preparation for later block creation. -pub struct OpenBlock<'x, 'y> { +pub struct OpenBlock<'x> { block: ExecutedBlock, engine: &'x Engine, - last_hashes: &'y LastHashes, + last_hashes: LastHashes, } /// Just like OpenBlock, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields, /// and collected the uncles. /// /// There is no function available to push a transaction. If you want that you'll need to `reopen()` it. -pub struct ClosedBlock<'x, 'y> { - open_block: OpenBlock<'x, 'y>, +pub struct ClosedBlock<'x> { + open_block: OpenBlock<'x>, uncle_bytes: Bytes, } @@ -169,9 +169,9 @@ pub struct SealedBlock { uncle_bytes: Bytes, } -impl<'x, 'y> OpenBlock<'x, 'y> { +impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -259,7 +259,7 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } /// Turn this into a `ClosedBlock`. A BlockChain must be provided in order to figure out the uncles. - pub fn close(self) -> ClosedBlock<'x, 'y> { + pub fn close(self) -> ClosedBlock<'x> { let mut s = self; s.engine.on_close_block(&mut s.block); s.block.base.header.transactions_root = ordered_trie_root(s.block.base.transactions.iter().map(|ref e| e.rlp_bytes().to_vec()).collect()); @@ -275,16 +275,16 @@ impl<'x, 'y> OpenBlock<'x, 'y> { } } -impl<'x, 'y> IsBlock for OpenBlock<'x, 'y> { +impl<'x> IsBlock for OpenBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.block } } -impl<'x, 'y> IsBlock for ClosedBlock<'x, 'y> { +impl<'x> IsBlock for ClosedBlock<'x> { fn block(&self) -> &ExecutedBlock { &self.open_block.block } } -impl<'x, 'y> ClosedBlock<'x, 'y> { - fn new(open_block: OpenBlock<'x, 'y>, uncle_bytes: Bytes) -> Self { +impl<'x> ClosedBlock<'x> { + fn new(open_block: OpenBlock<'x>, uncle_bytes: Bytes) -> Self { ClosedBlock { open_block: open_block, uncle_bytes: uncle_bytes, @@ -307,7 +307,7 @@ impl<'x, 'y> ClosedBlock<'x, 'y> { } /// Turn this back into an `OpenBlock`. - pub fn reopen(self) -> OpenBlock<'x, 'y> { self.open_block } + pub fn reopen(self) -> OpenBlock<'x> { self.open_block } /// Drop this object and return the underlieing database. pub fn drain(self) -> JournalDB { self.open_block.block.state.drop().1 } @@ -332,7 +332,7 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact<'x>(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { { if ::log::max_log_level() >= ::log::LogLevel::Trace { let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); @@ -350,20 +350,20 @@ pub fn enact<'x, 'y>(header: &Header, transactions: &[SignedTransaction], uncles } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes<'x, 'y>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_bytes<'x>(block_bytes: &[u8], engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified<'x, 'y>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: &'y LastHashes) -> Result, Error> { +pub fn enact_verified<'x>(block: &PreVerifiedBlock, engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result, Error> { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: &LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(header.seal()))) } @@ -384,7 +384,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); let _ = b.seal(vec![]); } @@ -398,14 +398,14 @@ mod tests { let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, vec![genesis_header.hash()], Address::zero(), vec![]).close().seal(vec![]).unwrap(); let orig_bytes = b.rlp_bytes(); let orig_db = b.drain(); let mut db_result = get_temp_journal_db(); let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); - let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, &vec![genesis_header.hash()]).unwrap(); + let e = enact_and_seal(&orig_bytes, engine.deref(), db, &genesis_header, vec![genesis_header.hash()]).unwrap(); assert_eq!(e.rlp_bytes(), orig_bytes); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index 3c44b97bf..2d57fbc3d 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -262,7 +262,6 @@ impl Client { fn check_and_close_block(&self, block: &PreVerifiedBlock) -> Result { let engine = self.engine.deref().deref(); let header = &block.header; - let header_hash = block.header.hash(); // Verify Block Family let verify_family_result = verify_block_family(&header, &block.bytes, engine, self.chain.read().unwrap().deref()); @@ -310,14 +309,13 @@ impl Client { let blocks = self.block_queue.write().unwrap().drain(max_blocks_to_import); for block in blocks { - let header = block.header; + let header = &block.header; if bad_blocks.contains(&header.parent_hash) { bad_blocks.insert(header.hash()); continue; } - let closed_block = self.check_and_close_block(&block); if let Err(_) = closed_block { bad_blocks.insert(header.hash()); @@ -350,7 +348,7 @@ impl Client { let bad_blocks = bad_blocks.into_iter().collect::>(); { - let block_queue = self.block_queue.write().unwrap(); + let mut block_queue = self.block_queue.write().unwrap(); block_queue.mark_as_bad(&bad_blocks); block_queue.mark_as_good(&good_blocks); } diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 8c411e7f0..54f02524d 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -282,7 +282,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let b = b.close(); assert_eq!(b.state().balance(&Address::zero()), U256::from_str("4563918244f40000").unwrap()); } @@ -295,7 +295,7 @@ mod tests { let mut db = db_result.take(); engine.spec().ensure_db_good(&mut db); let last_hashes = vec![genesis_header.hash()]; - let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, &last_hashes, Address::zero(), vec![]); + let mut b = OpenBlock::new(engine.deref(), db, &genesis_header, last_hashes, Address::zero(), vec![]); let mut uncle = Header::new(); let uncle_author = address_from_hex("ef2d6d194084c2de36e0dabfce45d046b37d1106"); uncle.author = uncle_author.clone(); From 08647282dff3999353ba7dbcc8c6bac046b1245c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 24 Feb 2016 17:01:29 +0100 Subject: [PATCH 16/48] Fixing mark_as_bad implementation --- ethcore/res/ethereum/tests | 2 +- ethcore/src/block_queue.rs | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index f32954b3d..3116f85a4 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit f32954b3ddb5af2dc3dc9ec6d9a28bee848fdf70 +Subproject commit 3116f85a499ceaf4dfdc46726060fc056e2d7829 diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index b802e3c26..ff20021f2 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -287,20 +287,23 @@ impl BlockQueue { /// Mark given block and all its children as bad. Stops verification. pub fn mark_as_bad(&mut self, hashes: &[H256]) { let mut verification_lock = self.verification.lock().unwrap(); - let mut verification = verification_lock.deref_mut(); - let mut new_verified = VecDeque::new(); + let mut processing = self.processing.write().unwrap(); + let mut verification = verification_lock.deref_mut(); + + verification.bad.reserve(hashes.len()); for hash in hashes { verification.bad.insert(hash.clone()); - self.processing.write().unwrap().remove(&hash); - for block in verification.verified.drain(..) { - if verification.bad.contains(&block.header.parent_hash) { - verification.bad.insert(block.header.hash()); - self.processing.write().unwrap().remove(&block.header.hash()); - } - else { - new_verified.push_back(block); - } + processing.remove(&hash); + } + + let mut new_verified = VecDeque::new(); + for block in verification.verified.drain(..) { + if verification.bad.contains(&block.header.parent_hash) { + verification.bad.insert(block.header.hash()); + processing.remove(&block.header.hash()); + } else { + new_verified.push_back(block); } } verification.verified = new_verified; From dd8652dbf41715dcb44e7fa4c835634b99c1d771 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 24 Feb 2016 21:17:29 +0300 Subject: [PATCH 17/48] u256 to inline assembly opt --- util/benches/bigint.rs | 37 +++++++++++++++++ util/src/lib.rs | 1 + util/src/uint.rs | 90 ++++++++++++++++++++++++++++++------------ 3 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 util/benches/bigint.rs diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs new file mode 100644 index 000000000..6ba30d88c --- /dev/null +++ b/util/benches/bigint.rs @@ -0,0 +1,37 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! benchmarking for rlp +//! should be started with: +//! ```bash +//! multirust run nightly cargo bench +//! ``` + +#![feature(test)] + +extern crate test; +extern crate ethcore_util; + +use test::{Bencher, black_box}; +use ethcore_util::uint::*; + +#[bench] +fn u256_first_degree(b: &mut test::Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256::zero(), |old, new| { old.overflowing_add(U256::from(new)).0 }) + }); +} diff --git a/util/src/lib.rs b/util/src/lib.rs index 2b7438cf3..1f04240dc 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,6 +16,7 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] +#![cfg_attr(feature="dev", feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings diff --git a/util/src/uint.rs b/util/src/uint.rs index 6490cbd9b..8266aff42 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,6 +51,64 @@ macro_rules! impl_map_from { } } +macro_rules! overflowing_add_regular { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + let $name(ref me) = $self_expr; + let $name(ref you) = $other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + let mut overflow = false; + + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + + if ret[i] < me[i] { + if i < $n_words - 1 { + carry[i + 1] = 1; + b_carry = true; + } else { + overflow = true; + } + } + } + if b_carry { + let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); + (ret, overflow) + } else { + ($name(ret), overflow) + } + }) +} + +macro_rules! overflowing_add_u256_asm { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + xor %al, %al + adc $9, %r8 + adc $10, %r9 + adc $11, %r10 + adc $12, %r11 + adc $$0, %al" + : "={r8}"(result[0]), "={r9}"(result[1]), "={r10}"(result[2]), "={r11}"(result[3]), "={al}"(overflow) + : "{r8}"(self_t[0]), "{r9}"(self_t[1]), "{r10}"(self_t[2]), "{r11}"(self_t[3]), "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + overflowing_add_regular!($name, $n_words, $self_expr, $other) + ) +} + macro_rules! overflowing { ($op: expr, $overflow: expr) => ( { @@ -297,32 +355,14 @@ macro_rules! construct_uint { (res, overflow) } + #[cfg(all(feature = "dev", target_arch = "x86_64"))] fn overflowing_add(self, other: $name) -> ($name, bool) { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - let mut overflow = false; + overflowing_add_u256_asm!($name, $n_words, self, other) + } - for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - - if ret[i] < me[i] { - if i < $n_words - 1 { - carry[i + 1] = 1; - b_carry = true; - } else { - overflow = true; - } - } - } - if b_carry { - let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); - (ret, overflow) - } else { - ($name(ret), overflow) - } + #[cfg(not(all(feature = "dev", target_arch = "x86_64")))] + fn overflowing_add(self, other: $name) -> ($name, bool) { + overflowing_add_regular!($name, $n_words, self, other) } fn overflowing_sub(self, other: $name) -> ($name, bool) { @@ -1171,8 +1211,6 @@ mod tests { ); } - - #[test] #[should_panic] pub fn uint256_mul_overflow_panic() { From 476bb85d414ab5ae0ee1d67435757a517a3ff430 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 24 Feb 2016 21:36:31 +0300 Subject: [PATCH 18/48] r m/r + setc/xor --- util/src/uint.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 8266aff42..db8792592 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -91,13 +91,13 @@ macro_rules! overflowing_add_u256_asm { unsafe { asm!(" xor %al, %al - adc $9, %r8 - adc $10, %r9 - adc $11, %r10 - adc $12, %r11 - adc $$0, %al" - : "={r8}"(result[0]), "={r9}"(result[1]), "={r10}"(result[2]), "={r11}"(result[3]), "={al}"(overflow) - : "{r8}"(self_t[0]), "{r9}"(self_t[1]), "{r10}"(self_t[2]), "{r11}"(self_t[3]), "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]) + adc $9, $0 + adc $10, $1 + adc $11, $2 + adc $12, $3 + adc $$0, %al" + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) : : ); From 7821505139c32d8326dae60efe9b6b63e8e7530f Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 24 Feb 2016 23:08:21 +0300 Subject: [PATCH 19/48] sub x64 optimize --- util/benches/bigint.rs | 11 +++++++++- util/src/uint.rs | 49 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 6ba30d88c..524d31508 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -29,9 +29,18 @@ use test::{Bencher, black_box}; use ethcore_util::uint::*; #[bench] -fn u256_first_degree(b: &mut test::Bencher) { +fn u256_add(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); (0..n).fold(U256::zero(), |old, new| { old.overflowing_add(U256::from(new)).0 }) }); } + +#[bench] +fn u256_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256::zero(), |old, new| { old.overflowing_add(U256::from(new)).0 }) + }); +} + diff --git a/util/src/uint.rs b/util/src/uint.rs index db8792592..147b83e42 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -41,6 +41,8 @@ use from_json::*; use rustc_serialize::hex::ToHex; use serde; +#[cfg_attr(x64_asm_optimizations, all(feature = "dev", target_arch = "x86_64"))] + macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { impl From<$from> for $thing { @@ -81,7 +83,7 @@ macro_rules! overflowing_add_regular { }) } -macro_rules! overflowing_add_u256_asm { +macro_rules! add_64x_optimized { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; @@ -90,12 +92,38 @@ macro_rules! overflowing_add_u256_asm { let overflow: u8; unsafe { asm!(" - xor %al, %al adc $9, $0 adc $10, $1 adc $11, $2 adc $12, $3 - adc $$0, %al" + setc %al" + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) + : + : + ); + } + (U256(result), overflow != 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + overflowing_add_regular!($name, $n_words, $self_expr, $other) + ) +} + +macro_rules! sub_64x_optimized { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + sbb $9, %r8 + sbb $10, %r9 + sbb $11, %r10 + sbb $12, %r11 + setb %al" : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) : @@ -355,16 +383,23 @@ macro_rules! construct_uint { (res, overflow) } - #[cfg(all(feature = "dev", target_arch = "x86_64"))] + /// Optimized instructions + #[cfg(x64_asm_optimizations)] + #[inline] fn overflowing_add(self, other: $name) -> ($name, bool) { - overflowing_add_u256_asm!($name, $n_words, self, other) + add_64x_optimized!($name, $n_words, self, other) } - - #[cfg(not(all(feature = "dev", target_arch = "x86_64")))] + #[cfg(not(x64_asm_optimizations))] fn overflowing_add(self, other: $name) -> ($name, bool) { overflowing_add_regular!($name, $n_words, self, other) } + #[cfg(x64_asm_optimizations)] + #[inline] + fn overflowing_sub(self, other: $name) -> ($name, bool) { + sub_64x_optimized!($name, $n_words, self, other) + } + #[cfg(not(x64_asm_optimizations))] fn overflowing_sub(self, other: $name) -> ($name, bool) { let res = overflowing!((!other).overflowing_add(From::from(1u64))); let res = overflowing!(self.overflowing_add(res)); From ccaa1946810e5566304f5e6e5cc2f33883f2a99b Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 02:00:34 +0300 Subject: [PATCH 20/48] mul, bench showtime --- util/benches/bigint.rs | 20 ++++- util/src/uint.rs | 161 +++++++++++++++++++++++++++++++++-------- 2 files changed, 150 insertions(+), 31 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 524d31508..38ce10a4a 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -21,6 +21,7 @@ //! ``` #![feature(test)] +#![feature(asm)] extern crate test; extern crate ethcore_util; @@ -40,7 +41,24 @@ fn u256_add(b: &mut Bencher) { fn u256_sub(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256::zero(), |old, new| { old.overflowing_add(U256::from(new)).0 }) + (0..n).fold(U256::zero(), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + }); +} + +#[bench] +fn u256_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256([12345u64, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_mul(U256::from(new)).0 }) + }); +} + + +#[bench] +fn u128_mul(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U128([12345u64, 0u64]), |old, new| { old.overflowing_mul(U128::from(new)).0 }) }); } diff --git a/util/src/uint.rs b/util/src/uint.rs index 147b83e42..38b4e4906 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -41,8 +41,6 @@ use from_json::*; use rustc_serialize::hex::ToHex; use serde; -#[cfg_attr(x64_asm_optimizations, all(feature = "dev", target_arch = "x86_64"))] - macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { impl From<$from> for $thing { @@ -53,7 +51,8 @@ macro_rules! impl_map_from { } } -macro_rules! overflowing_add_regular { +#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +macro_rules! uint_overflowing_add { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ let $name(ref me) = $self_expr; let $name(ref you) = $other; @@ -83,7 +82,8 @@ macro_rules! overflowing_add_regular { }) } -macro_rules! add_64x_optimized { +#[cfg(all(feature="dev", target_arch = "x86_64"))] +macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; @@ -110,7 +110,17 @@ macro_rules! add_64x_optimized { ) } -macro_rules! sub_64x_optimized { +#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +macro_rules! uint_overflowing_sub { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) +} + +#[cfg(all(feature="dev", target_arch = "x86_64"))] +macro_rules! uint_overflowing_sub { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; @@ -137,6 +147,119 @@ macro_rules! sub_64x_optimized { ) } +#[cfg(all(feature="dev", target_arch = "x86_64"))] +macro_rules! uint_overflowing_mul { + (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 4] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + unsafe { + asm!(" + mov $5, %rax + mulq $9 + mov %rax, %r8 + adc $6, %rdx + pushf + + mov %rdx, %rax + mulq $9 + popf + adc $$0, %rax + adc $7, %rdx + pushf + mov %rax, %r9 + + + mov %rdx, %rax + mulq $9 + popf + adc $$0, %rax + adc $8, %rdx + pushf + mov %rax, %r10 + + mov %rdx, %rax + mulq $9 + popf + adc $$0, %rax + mov %rax, %r11 + mov %rdx, %rcx + + mov $5, %rax + mulq $10 + adc %rax, %r9 + adc $6, %rdx + pushf + + mov %rdx, %rax + mulq $10 + popf + adc %rax, %r10 + adc $7, %rdx + pushf + + mov %rdx, %rax + mulq $10 + popf + adc %rax, %r11 + pushf + or %rax, %rcx + + mov $5, %rax + mulq $11 + popf + adc %rax, %r10 + adc $6, %rdx + pushf + + mov %rdx, %rax + mulq $11 + popf + adc %rax, %r11 + pushf + or %rdx, %rcx + + mov $5, %rax + mulq $12 + popf + adc %rax, %r11 + or %rdx, %rcx + " + : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), + /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) + + : /* $5 */ "m"(self_t[0]), /* $6 */ "m"(self_t[1]), /* $7 */ "m"(self_t[2]), + /* $8 */ "m"(self_t[3]), /* $9 */ "m"(other_t[0]), /* $10 */ "m"(other_t[1]), + /* $11 */ "m"(other_t[2]), /* $12 */ "m"(other_t[3]) + : "rax", "rdx" + : + + ); + } + (U256(result), overflow > 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( + overflowing_mul_regular!($name, $n_words, $self_expr, $other) + ) +} + +#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +macro_rules! uint_overflowing_mul { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut res = $name::from(0u64); + let mut overflow = false; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + let v = overflowing!($self_expr.overflowing_mul_u32(($other >> (32 * i)).low_u32()), overflow); + let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); + res = overflowing!(res.overflowing_add(res2), overflow); + } + (res, overflow) + }) +} + macro_rules! overflowing { ($op: expr, $overflow: expr) => ( { @@ -384,38 +507,16 @@ macro_rules! construct_uint { } /// Optimized instructions - #[cfg(x64_asm_optimizations)] - #[inline] fn overflowing_add(self, other: $name) -> ($name, bool) { - add_64x_optimized!($name, $n_words, self, other) - } - #[cfg(not(x64_asm_optimizations))] - fn overflowing_add(self, other: $name) -> ($name, bool) { - overflowing_add_regular!($name, $n_words, self, other) + uint_overflowing_add!($name, $n_words, self, other) } - #[cfg(x64_asm_optimizations)] - #[inline] fn overflowing_sub(self, other: $name) -> ($name, bool) { - sub_64x_optimized!($name, $n_words, self, other) - } - #[cfg(not(x64_asm_optimizations))] - fn overflowing_sub(self, other: $name) -> ($name, bool) { - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - let res = overflowing!(self.overflowing_add(res)); - (res, self < other) + uint_overflowing_sub!($name, $n_words, self, other) } fn overflowing_mul(self, other: $name) -> ($name, bool) { - let mut res = $name::from(0u64); - let mut overflow = false; - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow); - let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); - res = overflowing!(res.overflowing_add(res2), overflow); - } - (res, overflow) + uint_overflowing_mul!($name, $n_words, self, other) } fn overflowing_div(self, other: $name) -> ($name, bool) { From 0794049d18708a3d258df42cd43f51ce03a9ae33 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 02:05:59 +0300 Subject: [PATCH 21/48] fix naughty macros --- util/src/uint.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 38b4e4906..f27150199 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -53,6 +53,12 @@ macro_rules! impl_map_from { #[cfg(not(all(feature="dev", target_arch = "x86_64")))] macro_rules! uint_overflowing_add { + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_add_reg { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ let $name(ref me) = $self_expr; let $name(ref you) = $other; @@ -82,6 +88,7 @@ macro_rules! uint_overflowing_add { }) } + #[cfg(all(feature="dev", target_arch = "x86_64"))] macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ @@ -106,7 +113,7 @@ macro_rules! uint_overflowing_add { (U256(result), overflow != 0) }); ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( - overflowing_add_regular!($name, $n_words, $self_expr, $other) + uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) ) } @@ -142,9 +149,11 @@ macro_rules! uint_overflowing_sub { } (U256(result), overflow != 0) }); - ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( - overflowing_add_regular!($name, $n_words, $self_expr, $other) - ) + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let res = overflowing!((!$other).overflowing_add(From::from(1u64))); + let res = overflowing!($self_expr.overflowing_add(res)); + (res, $self_expr < $other) + }) } #[cfg(all(feature="dev", target_arch = "x86_64"))] @@ -241,12 +250,18 @@ macro_rules! uint_overflowing_mul { (U256(result), overflow > 0) }); ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( - overflowing_mul_regular!($name, $n_words, $self_expr, $other) + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) ) } #[cfg(not(all(feature="dev", target_arch = "x86_64")))] macro_rules! uint_overflowing_mul { + ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ + uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) + }) +} + +macro_rules! uint_overflowing_mul_reg { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut res = $name::from(0u64); let mut overflow = false; From da69ea51fe2c5da3b540fb07066588a7a4e27d1e Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 03:09:33 +0300 Subject: [PATCH 22/48] inline --- util/benches/bigint.rs | 12 ++++++++++-- util/src/uint.rs | 27 ++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 38ce10a4a..3a9c6d118 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -33,7 +33,15 @@ use ethcore_util::uint::*; fn u256_add(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256::zero(), |old, new| { old.overflowing_add(U256::from(new)).0 }) + (0..n).fold(U256::from(1234599u64), |old, new| { old.overflowing_add(U256::from(new)).0 }) + }); +} + +#[bench] +fn u256_uber_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U256::from(1234599u64), |old, new| { old.uber_add(U256::from(new)).0 }) }); } @@ -41,7 +49,7 @@ fn u256_add(b: &mut Bencher) { fn u256_sub(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256::zero(), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + (0..n).fold(U256::from(::std::u64::MAX), |old, new| { old.overflowing_sub(U256::from(new)).0 }) }); } diff --git a/util/src/uint.rs b/util/src/uint.rs index f27150199..8dd7d8638 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -97,17 +97,19 @@ macro_rules! uint_overflowing_add { let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; let overflow: u8; - unsafe { - asm!(" - adc $9, $0 - adc $10, $1 - adc $11, $2 - adc $12, $3 - setc %al" - : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) - : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) - : - : + unsafe { + asm!(" + adc $9, %r8 + adc $10, %r9 + adc $11, %r10 + adc $12, %r11 + setc %al + " + : "={r8}"(result[0]), "={r9}"(result[1]), "={r10}"(result[2]), "={r11}"(result[3]), "={al}"(overflow) + : "{r8}"(self_t[0]), "{r9}"(self_t[1]), "{r10}"(self_t[2]), "{r11}"(self_t[3]), + "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]) + : + : ); } (U256(result), overflow != 0) @@ -522,14 +524,17 @@ macro_rules! construct_uint { } /// Optimized instructions + #[inline(always)] fn overflowing_add(self, other: $name) -> ($name, bool) { uint_overflowing_add!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_sub(self, other: $name) -> ($name, bool) { uint_overflowing_sub!($name, $n_words, self, other) } + #[inline(always)] fn overflowing_mul(self, other: $name) -> ($name, bool) { uint_overflowing_mul!($name, $n_words, self, other) } From ae76a509dcc956ea329781bbc9cb6b6cc373580e Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 03:10:02 +0300 Subject: [PATCH 23/48] inline test --- util/benches/bigint.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 3a9c6d118..826d5023e 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -37,13 +37,6 @@ fn u256_add(b: &mut Bencher) { }); } -#[bench] -fn u256_uber_add(b: &mut Bencher) { - b.iter(|| { - let n = black_box(10000); - (0..n).fold(U256::from(1234599u64), |old, new| { old.uber_add(U256::from(new)).0 }) - }); -} #[bench] fn u256_sub(b: &mut Bencher) { From f17d893f53f2551d51e590cb4ce2d296750f4093 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 16:20:57 +0300 Subject: [PATCH 24/48] fixed mul, fixed register pref --- util/src/uint.rs | 159 +++++++++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 66 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 8dd7d8638..6869c3ec1 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -99,15 +99,15 @@ macro_rules! uint_overflowing_add { let overflow: u8; unsafe { asm!(" - adc $9, %r8 - adc $10, %r9 - adc $11, %r10 - adc $12, %r11 + adc $9, $0 + adc $10, $1 + adc $11, $2 + adc $12, $3 setc %al " - : "={r8}"(result[0]), "={r9}"(result[1]), "={r10}"(result[2]), "={r11}"(result[3]), "={al}"(overflow) - : "{r8}"(self_t[0]), "{r9}"(self_t[1]), "{r10}"(self_t[2]), "{r11}"(self_t[3]), - "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]) + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) : : ); @@ -138,10 +138,10 @@ macro_rules! uint_overflowing_sub { let overflow: u8; unsafe { asm!(" - sbb $9, %r8 - sbb $10, %r9 - sbb $11, %r10 - sbb $12, %r11 + sbb $9, $0 + sbb $10, $1 + sbb $11, $2 + sbb $12, $3 setb %al" : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) @@ -168,76 +168,103 @@ macro_rules! uint_overflowing_mul { let overflow: u8; unsafe { asm!(" + clc mov $5, %rax mulq $9 - mov %rax, %r8 - adc $6, %rdx - pushf + mov %rax, $0 + mov %rdx, $1 - mov %rdx, %rax + mov $6, %rax + mulq $9 + clc + adc %rax, $1 + mov %rdx, $2 + + mov $5, %rax + pushf + mulq $10 + popf + adc %rax, $1 + adc %rdx, $2 + + mov $6, %rax + mulq $10 + clc + adc %rax, $2 + mov %rdx, $3 + + mov $7, %rax + mulq $9 + clc + adc %rax, $2 + adc %rdx, $3 + + mov $5, %rax + mulq $11 + clc + adc %rax, $2 + adc %rdx, $3 + + mov $8, %rax + pushf mulq $9 popf - adc $$0, %rax - adc $7, %rdx - pushf - mov %rax, %r9 - - - mov %rdx, %rax - mulq $9 - popf - adc $$0, %rax - adc $8, %rdx - pushf - mov %rax, %r10 - - mov %rdx, %rax - mulq $9 - popf - adc $$0, %rax - mov %rax, %r11 + adc %rax, $3 + adc $$0, %rdx mov %rdx, %rcx + clc - mov $5, %rax - mulq $10 - adc %rax, %r9 - adc $6, %rdx + mov $7, %rax pushf - - mov %rdx, %rax mulq $10 popf - adc %rax, %r10 - adc $7, %rdx - pushf - - mov %rdx, %rax - mulq $10 - popf - adc %rax, %r11 - pushf - or %rax, %rcx - - mov $5, %rax - mulq $11 - popf - adc %rax, %r10 - adc $6, %rdx - pushf - - mov %rdx, %rax - mulq $11 - popf - adc %rax, %r11 - pushf + adc %rax, $3 + adc $$0, %rdx or %rdx, %rcx + clc + + mov $6, %rax + pushf + mulq $11 + popf + adc %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + clc mov $5, %rax + pushf mulq $12 popf - adc %rax, %r11 - or %rdx, %rcx - " + adc %rax, $3 + adc $$0, %rdx + or %rdx, %rcx + clc + + cmpq $$0, %rcx + jne 2f + + mov $8, %rax + cmpq $$0, %rax + setz %cl + + mov $7, %rax + cmpq $$0, %rax + sete %dl + or %dl, %cl + + mov $3, %rax + cmpq $$0, %rax + sete %dl + + mov $2, %rax + cmpq $$0, %rax + sete %bl + or %bl, %dl + + and %dl, %cl + + 2: " : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) From 5467b06c4f845bd8fde8adf216dec75deb4dbea6 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 16:40:36 +0300 Subject: [PATCH 25/48] fix bench iter --- util/benches/bigint.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 826d5023e..a22edcfbc 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -33,7 +33,7 @@ use ethcore_util::uint::*; fn u256_add(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256::from(1234599u64), |old, new| { old.overflowing_add(U256::from(new)).0 }) + (0..n).fold(U256([12345u64, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_add(U256::from(new)).0 }) }); } @@ -42,7 +42,7 @@ fn u256_add(b: &mut Bencher) { fn u256_sub(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256::from(::std::u64::MAX), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + (0..n).fold(U256([::std::u64::MAX, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_sub(U256::from(new)).0 }) }); } From fb5779a00eeeef78228ec61e09331fbed503293a Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 16:55:03 +0300 Subject: [PATCH 26/48] specific feature for asm opt --- util/Cargo.toml | 1 + util/src/lib.rs | 2 +- util/src/uint.rs | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/util/Cargo.toml b/util/Cargo.toml index 6d2ebcd9b..e2e91eb4b 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -39,6 +39,7 @@ target_info = "0.1" [features] default = [] dev = ["clippy"] +x64asm = [] [build-dependencies] vergen = "*" diff --git a/util/src/lib.rs b/util/src/lib.rs index 1f04240dc..d0c74af10 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -16,7 +16,7 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] -#![cfg_attr(feature="dev", feature(asm))] +#![cfg_attr(feature="x64asm", feature(asm))] #![cfg_attr(feature="dev", plugin(clippy))] // Clippy settings diff --git a/util/src/uint.rs b/util/src/uint.rs index 6869c3ec1..98c16ab90 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,7 +51,7 @@ macro_rules! impl_map_from { } } -#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] macro_rules! uint_overflowing_add { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) @@ -89,7 +89,7 @@ macro_rules! uint_overflowing_add_reg { } -#[cfg(all(feature="dev", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch = "x86_64"))] macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -119,7 +119,7 @@ macro_rules! uint_overflowing_add { ) } -#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] macro_rules! uint_overflowing_sub { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let res = overflowing!((!$other).overflowing_add(From::from(1u64))); @@ -128,7 +128,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="dev", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch = "x86_64"))] macro_rules! uint_overflowing_sub { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -158,7 +158,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="dev", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch = "x86_64"))] macro_rules! uint_overflowing_mul { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -283,7 +283,7 @@ macro_rules! uint_overflowing_mul { ) } -#[cfg(not(all(feature="dev", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] macro_rules! uint_overflowing_mul { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) From 7525ff23cf1802cbeb6e4d51394eac3974de4c70 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 17:59:08 +0300 Subject: [PATCH 27/48] removed artefact cls/pushf/popf --- util/src/uint.rs | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 98c16ab90..bebaade22 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -168,7 +168,6 @@ macro_rules! uint_overflowing_mul { let overflow: u8; unsafe { asm!(" - clc mov $5, %rax mulq $9 mov %rax, $0 @@ -176,77 +175,59 @@ macro_rules! uint_overflowing_mul { mov $6, %rax mulq $9 - clc - adc %rax, $1 + add %rax, $1 mov %rdx, $2 mov $5, %rax - pushf mulq $10 - popf - adc %rax, $1 + add %rax, $1 adc %rdx, $2 mov $6, %rax mulq $10 - clc - adc %rax, $2 + add %rax, $2 mov %rdx, $3 mov $7, %rax mulq $9 - clc - adc %rax, $2 + add %rax, $2 adc %rdx, $3 mov $5, %rax mulq $11 - clc - adc %rax, $2 + add %rax, $2 adc %rdx, $3 mov $8, %rax - pushf mulq $9 - popf adc %rax, $3 adc $$0, %rdx mov %rdx, %rcx - clc mov $7, %rax - pushf mulq $10 - popf - adc %rax, $3 + add %rax, $3 adc $$0, %rdx or %rdx, %rcx - clc mov $6, %rax - pushf mulq $11 - popf - adc %rax, $3 + add %rax, $3 adc $$0, %rdx or %rdx, %rcx - clc mov $5, %rax - pushf mulq $12 - popf - adc %rax, $3 + add %rax, $3 adc $$0, %rdx or %rdx, %rcx - clc cmpq $$0, %rcx jne 2f mov $8, %rax cmpq $$0, %rax - setz %cl + sete %cl mov $7, %rax cmpq $$0, %rax @@ -264,7 +245,8 @@ macro_rules! uint_overflowing_mul { and %dl, %cl - 2: " + 2: + " : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) From 864e7540742ebc1c408e9e2de57f96eee28d7c5b Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 18:02:08 +0300 Subject: [PATCH 28/48] overflowing_sub in sub --- util/src/uint.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index bebaade22..6793376a0 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -765,9 +765,9 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - panic_on_overflow!(self < other); - let res = overflowing!((!other).overflowing_add(From::from(1u64))); - overflowing!(self.overflowing_add(res)) + let (result, overflow) = self.overflowing_sub(other); + panic_on_overflow!(overflow); + result } } From 5d22ad3fc8abe4617684213833550207ddca2c6b Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 18:10:33 +0300 Subject: [PATCH 29/48] counter jump better --- util/src/uint.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 6793376a0..f4d5b5b76 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -222,8 +222,7 @@ macro_rules! uint_overflowing_mul { adc $$0, %rdx or %rdx, %rcx - cmpq $$0, %rcx - jne 2f + jrcxz 2f mov $8, %rax cmpq $$0, %rax @@ -234,6 +233,8 @@ macro_rules! uint_overflowing_mul { sete %dl or %dl, %cl + jrcxz 2f + mov $3, %rax cmpq $$0, %rax sete %dl From 2ee4a0c8c6ff29d6bbbc15752778590364d9bdb3 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 18:16:08 +0300 Subject: [PATCH 30/48] mistake of ne/jcxz --- util/src/uint.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index f4d5b5b76..6793376a0 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -222,7 +222,8 @@ macro_rules! uint_overflowing_mul { adc $$0, %rdx or %rdx, %rcx - jrcxz 2f + cmpq $$0, %rcx + jne 2f mov $8, %rax cmpq $$0, %rax @@ -233,8 +234,6 @@ macro_rules! uint_overflowing_mul { sete %dl or %dl, %cl - jrcxz 2f - mov $3, %rax cmpq $$0, %rax sete %dl From 600859ed04acd3650868f35bf4a1add4f983702d Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 19:58:09 +0300 Subject: [PATCH 31/48] [ci skip] flush --- util/src/uint.rs | 48 +++++++++++++----------------------------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 6793376a0..8e9172a04 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -165,7 +165,7 @@ macro_rules! uint_overflowing_mul { let self_t: &[u64; 4] = unsafe { &mem::transmute($self_expr) }; let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; - let overflow: u8; + let overflow: u64; unsafe { asm!(" mov $5, %rax @@ -222,25 +222,25 @@ macro_rules! uint_overflowing_mul { adc $$0, %rdx or %rdx, %rcx - cmpq $$0, %rcx + cmpq $$0, %rcx jne 2f mov $8, %rax cmpq $$0, %rax - sete %cl + setne %cl mov $7, %rax cmpq $$0, %rax - sete %dl + setne %dl or %dl, %cl mov $3, %rax cmpq $$0, %rax - sete %dl + setne %dl mov $2, %rax cmpq $$0, %rax - sete %bl + setne %bl or %bl, %dl and %dl, %cl @@ -253,7 +253,7 @@ macro_rules! uint_overflowing_mul { : /* $5 */ "m"(self_t[0]), /* $6 */ "m"(self_t[1]), /* $7 */ "m"(self_t[2]), /* $8 */ "m"(self_t[3]), /* $9 */ "m"(other_t[0]), /* $10 */ "m"(other_t[1]), /* $11 */ "m"(other_t[2]), /* $12 */ "m"(other_t[3]) - : "rax", "rdx" + : "rax", "rdx", "rbx" : ); @@ -740,23 +740,8 @@ macro_rules! construct_uint { type Output = $name; fn add(self, other: $name) -> $name { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - for i in 0..$n_words { - if i < $n_words - 1 { - ret[i] = me[i].wrapping_add(you[i]); - if ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; - } - } else { - ret[i] = me[i] + you[i]; - } - } - if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + let (result, _) = self.overflowing_add(other); + result } } @@ -765,8 +750,7 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - let (result, overflow) = self.overflowing_sub(other); - panic_on_overflow!(overflow); + let (result, _) = self.overflowing_sub(other); result } } @@ -775,15 +759,9 @@ macro_rules! construct_uint { type Output = $name; fn mul(self, other: $name) -> $name { - let mut res = $name::from(0u64); - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - let v = self.mul_u32((other >> (32 * i)).low_u32()); - let (r, overflow) = v.overflowing_shl(32 * i as u32); - panic_on_overflow!(overflow); - res = res + r; - } - res + let (result, overflow) = self.overflowing_mul(other); + panic_on_overflow!(overflow); + result } } From e946e2ab183f8c1c60d88769602e654911244396 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 22:27:22 +0300 Subject: [PATCH 32/48] epic mul overflow bug --- util/src/uint.rs | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 8e9172a04..f9d9b4af8 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -225,25 +225,17 @@ macro_rules! uint_overflowing_mul { cmpq $$0, %rcx jne 2f - mov $8, %rax - cmpq $$0, %rax - setne %cl + popcnt $8, %rcx + popcnt $7, %rax + add %rax, %rcx + jrcxz 2f - mov $7, %rax - cmpq $$0, %rax - setne %dl - or %dl, %cl + popcnt $12, %rcx + popcnt $11, %rax + add %rax, %rcx + jrcxz 2f - mov $3, %rax - cmpq $$0, %rax - setne %dl - - mov $2, %rax - cmpq $$0, %rax - setne %bl - or %bl, %dl - - and %dl, %cl + mov $$1, %rcx 2: " @@ -740,7 +732,8 @@ macro_rules! construct_uint { type Output = $name; fn add(self, other: $name) -> $name { - let (result, _) = self.overflowing_add(other); + let (result, overflow) = self.overflowing_add(other); + panic_on_overflow!(overflow); result } } @@ -750,7 +743,8 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - let (result, _) = self.overflowing_sub(other); + let (result, overflow) = self.overflowing_sub(other); + panic_on_overflow!(overflow); result } } From 4b0ec642995206fcef7cca18b8a3cec733813be3 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Thu, 25 Feb 2016 22:48:34 +0300 Subject: [PATCH 33/48] random init for benches --- util/benches/bigint.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index a22edcfbc..3b2012bc7 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -25,6 +25,7 @@ extern crate test; extern crate ethcore_util; +extern crate rand; use test::{Bencher, black_box}; use ethcore_util::uint::*; @@ -33,7 +34,7 @@ use ethcore_util::uint::*; fn u256_add(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256([12345u64, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_add(U256::from(new)).0 }) + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_add(U256::from(new)).0 }) }); } @@ -42,7 +43,7 @@ fn u256_add(b: &mut Bencher) { fn u256_sub(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256([::std::u64::MAX, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_sub(U256::from(new)).0 }) + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_sub(U256::from(new)).0 }) }); } @@ -50,7 +51,7 @@ fn u256_sub(b: &mut Bencher) { fn u256_mul(b: &mut Bencher) { b.iter(|| { let n = black_box(10000); - (0..n).fold(U256([12345u64, 0u64, 0u64, 0u64]), |old, new| { old.overflowing_mul(U256::from(new)).0 }) + (0..n).fold(U256([rand::random::(), rand::random::(), rand::random::(), rand::random::()]), |old, new| { old.overflowing_mul(U256::from(new)).0 }) }); } From f29417eea91f689fe57d3d8b06e1921b93235291 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 14:50:55 +0300 Subject: [PATCH 34/48] allow dead code for macros expansion --- util/src/uint.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/src/uint.rs b/util/src/uint.rs index f9d9b4af8..ca727190b 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -588,6 +588,7 @@ macro_rules! construct_uint { } impl $name { + #[allow(dead_code)] // not used when multiplied with inline assembly /// Multiplication by u32 fn mul_u32(self, other: u32) -> Self { let $name(ref arr) = self; @@ -609,6 +610,7 @@ macro_rules! construct_uint { $name(ret) + $name(carry) } + #[allow(dead_code)] // not used when multiplied with inline assembly /// Overflowing multiplication by u32 fn overflowing_mul_u32(self, other: u32) -> (Self, bool) { let $name(ref arr) = self; From 80d60cedf6f43b7cc3d03485f1c5be3202dec5f4 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 26 Feb 2016 13:27:05 +0100 Subject: [PATCH 35/48] Removed rocksdb from build scripts and instructions --- README.md | 25 +-------------------- ethcore/src/lib.rs | 13 ++--------- install-deps.sh | 55 +--------------------------------------------- install-parity.sh | 50 ++++++++++++++++------------------------- 4 files changed, 23 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index 4df7cad34..575234622 100644 --- a/README.md +++ b/README.md @@ -18,17 +18,9 @@ ##### Ubuntu 14.04, 15.04, 15.10 ```bash -# install rocksdb -add-apt-repository ppa:ethcore/ethcore -apt-get update -apt-get install -y --force-yes librocksdb-dev - # install multirust curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes -# install beta -multirust update beta - # download and build parity git clone https://github.com/ethcore/parity cd parity @@ -43,20 +35,9 @@ cargo build --release ##### Other Linux ```bash -# install rocksdb -git clone --tag v4.1 --depth=1 https://github.com/facebook/rocksdb.git -cd rocksdb -make shared_lib -sudo cp -a librocksdb.so* /usr/lib -sudo ldconfig -cd .. - # install rust beta curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes -# install beta -multirust update beta - # download and build parity git clone https://github.com/ethcore/parity cd parity @@ -71,14 +52,10 @@ cargo build --release ##### OSX with Homebrew ```bash -# install rocksdb && multirust +# install multirust brew update -brew install rocksdb brew install multirust -# install beta -multirust update beta - # download and build parity git clone https://github.com/ethcore/parity cd parity diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index d4786bc21..5eb3beeb2 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -40,23 +40,17 @@ //! - Ubuntu 14.04 and later: //! //! ```bash -//! # install rocksdb -//! add-apt-repository "deb http://ppa.launchpad.net/giskou/librocksdb/ubuntu trusty main" -//! apt-get update -//! apt-get install -y --force-yes librocksdb //! //! # install multirust //! curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` //! @@ -65,18 +59,15 @@ //! ```bash //! # install rocksdb && multirust //! brew update -//! brew install rocksdb //! brew install multirust //! -//! # install nightly and make it default -//! multirust update nightly && multirust default nightly -//! //! # export rust LIBRARY_PATH //! export LIBRARY_PATH=/usr/local/lib //! //! # download and build parity //! git clone https://github.com/ethcore/parity //! cd parity +//! multirust override beta //! cargo build --release //! ``` diff --git a/install-deps.sh b/install-deps.sh index f5e1e44cf..6cc7de002 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -342,8 +342,6 @@ function run_installer() exe brew update echo - info "Installing rocksdb" - exe brew install rocksdb info "Installing multirust" exe brew install multirust sudo multirust default beta @@ -391,7 +389,6 @@ function run_installer() linux_version find_multirust - find_rocksdb find_curl find_git @@ -402,21 +399,6 @@ function run_installer() find_sudo } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "apt-get" - isRocksDB=true - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==> librocksdb:${reset}$n" - fi - } - function find_multirust() { depCount=$((depCount+2)) @@ -562,34 +544,6 @@ function run_installer() fi } - function ubuntu_rocksdb_installer() - { - sudo apt-get update -qq - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb-dev librocksdb - } - - function linux_rocksdb_installer() - { - if [[ $isUbuntu == true ]]; then - ubuntu_rocksdb_installer - else - oldpwd=`pwd` - cd /tmp - exe git clone --branch v4.2 --depth=1 https://github.com/facebook/rocksdb.git - cd rocksdb - exe make shared_lib - sudo cp -a librocksdb.so* /usr/lib - sudo ldconfig - cd /tmp - rm -rf /tmp/rocksdb - cd $oldpwd - fi - } - function linux_installer() { if [[ $isGCC == false || $isGit == false || $isMake == false || $isCurl == false ]]; then @@ -610,12 +564,6 @@ function run_installer() echo fi - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - linux_rocksdb_installer - echo - fi - if [[ $isMultirust == false ]]; then info "Installing multirust..." if [[ $isSudo == false ]]; then @@ -655,10 +603,9 @@ function run_installer() find_git find_make find_gcc - find_rocksdb find_multirust - if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isRocksDB == false || $isMultirustBeta == false ]]; then + if [[ $isCurl == false || $isGit == false || $isMake == false || $isGCC == false || $isMultirustBeta == false ]]; then abort_install fi fi diff --git a/install-parity.sh b/install-parity.sh index 60d3471d5..439700306 100755 --- a/install-parity.sh +++ b/install-parity.sh @@ -236,14 +236,29 @@ function run_installer() { linux_version - find_rocksdb - find_curl find_apt find_sudo } + function find_git() + { + depCount=$((depCount+1)) + GIT_PATH=`which git 2>/dev/null` + + if [[ -f $GIT_PATH ]] + then + depFound=$((depFound+1)) + check "git" + isGit=true + else + uncheck "git is missing" + isGit=false + INSTALL_FILES+="${blue}${dim}==> git:${reset}${n}" + fi + } + function find_brew() { BREW_PATH=`which brew 2>/dev/null` @@ -333,20 +348,6 @@ function run_installer() fi } - function find_rocksdb() - { - depCount=$((depCount+1)) - if [[ $(ldconfig -v 2>/dev/null | grep rocksdb | wc -l) == 1 ]]; then - depFound=$((depFound+1)) - check "librocksdb" - isRocksDB=true - else - uncheck "librocksdb is missing" - isRocksDB=false - INSTALL_FILES+="${blue}${dim}==>${reset}\tlibrocksdb${n}" - fi - } - function find_apt() { depCount=$((depCount+1)) @@ -386,10 +387,9 @@ function run_installer() info "Verifying installation" if [[ $OS_TYPE == "linux" ]]; then - find_rocksdb find_apt - if [[ $isRocksDB == false || $isApt == false ]]; then + if [[ $isApt == false ]]; then abortInstall fi fi @@ -397,23 +397,11 @@ function run_installer() function linux_deps_installer() { - if [[ $isRocksDB == false || $isCurl == false ]]; then + if [[ $isCurl == false ]]; then info "Preparing apt..." sudo apt-get update -qq echo fi - - if [[ $isRocksDB == false ]]; then - info "Installing rocksdb..." - - sudo apt-get install -qq -y software-properties-common - sudo apt-add-repository -y ppa:ethcore/ethcore - sudo apt-get -f -y install - sudo apt-get update -qq - sudo apt-get install -qq -y librocksdb - - echo - fi if [[ $isCurl == false ]]; then info "Installing curl..." From e95538f3ec716af0e051b6ed88761105b598defd Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 15:56:55 +0300 Subject: [PATCH 36/48] [ci skip] style fixes, multipart add test --- util/src/uint.rs | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index ca727190b..82d3afe97 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -51,7 +51,7 @@ macro_rules! impl_map_from { } } -#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] macro_rules! uint_overflowing_add { ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) @@ -89,7 +89,7 @@ macro_rules! uint_overflowing_add_reg { } -#[cfg(all(feature="x64asm", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch="x86_64"))] macro_rules! uint_overflowing_add { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -119,7 +119,7 @@ macro_rules! uint_overflowing_add { ) } -#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] macro_rules! uint_overflowing_sub { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let res = overflowing!((!$other).overflowing_add(From::from(1u64))); @@ -128,7 +128,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch="x86_64"))] macro_rules! uint_overflowing_sub { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -158,7 +158,7 @@ macro_rules! uint_overflowing_sub { }) } -#[cfg(all(feature="x64asm", target_arch = "x86_64"))] +#[cfg(all(feature="x64asm", target_arch="x86_64"))] macro_rules! uint_overflowing_mul { (U256, $n_words: expr, $self_expr: expr, $other: expr) => ({ let mut result: [u64; 4] = unsafe { mem::uninitialized() }; @@ -222,7 +222,7 @@ macro_rules! uint_overflowing_mul { adc $$0, %rdx or %rdx, %rcx - cmpq $$0, %rcx + cmpq $$0, %rcx jne 2f popcnt $8, %rcx @@ -257,7 +257,7 @@ macro_rules! uint_overflowing_mul { ) } -#[cfg(not(all(feature="x64asm", target_arch = "x86_64")))] +#[cfg(not(all(feature="x64asm", target_arch="x86_64")))] macro_rules! uint_overflowing_mul { ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ uint_overflowing_mul_reg!($name, $n_words, $self_expr, $other) @@ -1468,5 +1468,26 @@ mod tests { fn display_uint_zero() { assert_eq!(format!("{}", U256::from(0)), "0"); } + + + #[test] + fn u256_multi_adds() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_add(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_add(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 2])); + + let (result, overflow) = U256([0, 0, 2, 1]).overflowing_add(U256([0, 0, 3, 1])); + assert_eq!(result, U256([0, 0, 5, 2])); + assert!(!overflow); + + let (_, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_add(U256([0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + } } From 228e3fefe02445b39ce227d182331483beb90dd5 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 16:03:04 +0300 Subject: [PATCH 37/48] [ci skip] multipart sub test --- util/src/uint.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/util/src/uint.rs b/util/src/uint.rs index 82d3afe97..245381b4b 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -1489,5 +1489,32 @@ mod tests { let (_, overflow) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_add(U256([0, 0, 0, ::std::u64::MAX])); assert!(overflow); } + + + #[test] + fn u256_multi_subs() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_sub(U256([0, 0, 0, 0])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 0, 1])); + assert_eq!(result, U256([0, 0, 0, 0])); + + let (_, overflow) = U256([0, 0, 2, 1]).overflowing_sub(U256([0, 0, 3, 1])); + assert!(overflow); + + let (result, overflow) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_sub(U256([::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2, ::std::u64::MAX/2])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1, ::std::u64::MAX/2+1]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([0, 0, 1, 0])); + assert!(!overflow); + assert_eq!(U256([0, 0, ::std::u64::MAX, 0]), result); + + let (result, overflow) = U256([0, 0, 0, 1]).overflowing_sub(U256([1, 0, 0, 0])); + assert!(!overflow); + assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); + } + } From 3858a2011fb765dbb785f15773f21bb4578548f6 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 16:12:47 +0300 Subject: [PATCH 38/48] [ci skip] mul multipart tests --- util/src/uint.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/util/src/uint.rs b/util/src/uint.rs index 245381b4b..a34742bd9 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -1516,5 +1516,51 @@ mod tests { assert_eq!(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, 0]), result); } + + #[test] + fn u256_multi_muls() { + let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([1, 0, 0, 0]), result); + + let (result, _) = U256([5, 0, 0, 0]).overflowing_mul(U256([5, 0, 0, 0])); + assert_eq!(U256([25, 0, 0, 0]), result); + + let (result, _) = U256([0, 5, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 25, 0]), result); + + let (result, _) = U256([0, 0, 0, 1]).overflowing_mul(U256([1, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 1]), result); + + let (result, _) = U256([0, 0, 0, 5]).overflowing_mul(U256([2, 0, 0, 0])); + assert_eq!(U256([0, 0, 0, 10]), result); + + let (result, _) = U256([0, 0, 1, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 0, 0, 5]), result); + + let (result, _) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 5, 0, 0])); + assert_eq!(U256([0, 10, 0, 0]), result); + + let (result, _) = U256([::std::u64::MAX, 0, 0, 0]).overflowing_mul(U256([::std::u64::MAX, 0, 0, 0])); + assert_eq!(U256([1, ::std::u64::MAX-1, 0, 0]), result); + + let (result, _) = U256([0, 0, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, 0]), result); + + let (result, _) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert_eq!(U256([0, 0, 0, ::std::u64::MAX]), result); + + let (result, _) = U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_mul(U256([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert_eq!(U256([1, 0, 0, 0]), result); + } + + + } From 023c6236500b82f89110c0f482e15b23a559cbb5 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 16:19:55 +0300 Subject: [PATCH 39/48] mul overflow multipart test --- util/src/uint.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/util/src/uint.rs b/util/src/uint.rs index a34742bd9..98541fe33 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -1560,7 +1560,34 @@ mod tests { assert_eq!(U256([1, 0, 0, 0]), result); } + #[test] + fn u256_multi_muls_overflow() { + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); + assert!(!overflow); + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(!overflow); + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([0, 1, 0, ::std::u64::MAX]).overflowing_mul(U256([0, 1, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U256([0, ::std::u64::MAX, 0, 0]).overflowing_mul(U256([0, ::std::u64::MAX, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([1, 0, 0, 0]).overflowing_mul(U256([10, 0, 0, 0])); + assert!(!overflow); + + let (_, overflow) = U256([2, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX / 2])); + assert!(!overflow); + + let (_, overflow) = U256([0, 0, 8, 0]).overflowing_mul(U256([0, 0, 7, 0])); + assert!(overflow); + } } From 5013c4d1f1efc38e2237ed691e48f9e3f5e54aa0 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 16:50:12 +0300 Subject: [PATCH 40/48] naughty overflow bug fixed --- util/src/uint.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index 98541fe33..b4940cfb8 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -226,16 +226,30 @@ macro_rules! uint_overflowing_mul { jne 2f popcnt $8, %rcx - popcnt $7, %rax - add %rax, %rcx - jrcxz 2f + jrcxz 12f popcnt $12, %rcx popcnt $11, %rax add %rax, %rcx - jrcxz 2f + popcnt $10, %rax + add %rax, %rcx + jmp 2f - mov $$1, %rcx + 12: + popcnt $12, %rcx + jrcxz 11f + + popcnt $7, %rcx + popcnt $6, %rax + add %rax, %rcx + + cmpq $$0, %rcx + jne 2f + + 11: + popcnt $11, %rcx + jrcxz 2f + popcnt $7, %rcx 2: " @@ -1569,7 +1583,7 @@ mod tests { assert!(!overflow); let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 0, 0, ::std::u64::MAX])); - assert!(!overflow); + assert!(overflow); let (_, overflow) = U256([0, 1, 0, 0]).overflowing_mul(U256([0, 1, 0, 0])); assert!(!overflow); From 3750a8964c0424387cdf545a6d1a4f7dd5035147 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 17:48:50 +0300 Subject: [PATCH 41/48] removed redundant clones --- util/src/keys/geth_import.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/src/keys/geth_import.rs b/util/src/keys/geth_import.rs index 61df88589..2ee3c987c 100644 --- a/util/src/keys/geth_import.rs +++ b/util/src/keys/geth_import.rs @@ -63,12 +63,13 @@ pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) let mut buf = String::new(); try!(file.read_to_string(&mut buf)); - let mut json = match Json::from_str(&buf) { - Ok(parsed_json) => try!(parsed_json.as_object().ok_or(ImportError::FormatError)).clone(), + let mut json_result = Json::from_str(&buf); + let mut json = match json_result { + Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::FormatError)), Err(_) => { return Err(ImportError::FormatError); } }; let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone(); - json.insert("crypto".to_owned(), Json::Object(crypto_object.clone())); + json.insert("crypto".to_owned(), Json::Object(crypto_object)); json.remove("Crypto"); match KeyFileContent::load(&Json::Object(json.clone())) { Ok(key_file) => try!(secret_store.import_key(key_file)), From fd63fa6836489039756f105086b8b41fac793dad Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Feb 2016 17:27:56 +0100 Subject: [PATCH 42/48] Update block.rs --- ethcore/src/block.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 9051b6d9a..70c6c2fb2 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -144,7 +144,7 @@ impl IsBlock for ExecutedBlock { /// Block that is ready for transactions to be added. /// -/// It's a bit like a Vec, eccept that whenever a transaction is pushed, we execute it and +/// It's a bit like a Vec, except that whenever a transaction is pushed, we execute it and /// maintain the system `state()`. We also archive execution receipts in preparation for later block creation. pub struct OpenBlock<'x> { block: ExecutedBlock, From f118e30b209074236c4593693d3285c032ed2bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 26 Feb 2016 19:56:32 +0100 Subject: [PATCH 43/48] Renaming variables to more descriptive --- ethcore/src/block_queue.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index ff20021f2..62763386f 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -285,14 +285,14 @@ impl BlockQueue { } /// Mark given block and all its children as bad. Stops verification. - pub fn mark_as_bad(&mut self, hashes: &[H256]) { + pub fn mark_as_bad(&mut self, block_hashes: &[H256]) { let mut verification_lock = self.verification.lock().unwrap(); let mut processing = self.processing.write().unwrap(); let mut verification = verification_lock.deref_mut(); - verification.bad.reserve(hashes.len()); - for hash in hashes { + verification.bad.reserve(block_hashes.len()); + for hash in block_hashes { verification.bad.insert(hash.clone()); processing.remove(&hash); } @@ -310,10 +310,10 @@ impl BlockQueue { } /// Mark given block as processed - pub fn mark_as_good(&mut self, hashes: &[H256]) { + pub fn mark_as_good(&mut self, block_hashes: &[H256]) { let mut processing = self.processing.write().unwrap(); - for h in hashes { - processing.remove(&h); + for hash in block_hashes { + processing.remove(&hash); } } From 9585138f20fdf74c327c484fa7e1a1a24f4393e2 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 23:24:04 +0300 Subject: [PATCH 44/48] add/sub 512 --- util/src/uint.rs | 179 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 159 insertions(+), 20 deletions(-) diff --git a/util/src/uint.rs b/util/src/uint.rs index b4940cfb8..5a2730126 100644 --- a/util/src/uint.rs +++ b/util/src/uint.rs @@ -97,23 +97,69 @@ macro_rules! uint_overflowing_add { let other_t: &[u64; 4] = unsafe { &mem::transmute($other) }; let overflow: u8; - unsafe { - asm!(" - adc $9, $0 - adc $10, $1 - adc $11, $2 - adc $12, $3 - setc %al - " - : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) - : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + unsafe { + asm!(" + add $9, $0 + adc $10, $1 + adc $11, $2 + adc $12, $3 + setc %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) - : - : + : + : ); } (U256(result), overflow != 0) }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + add $15, $0 + adc $16, $1 + adc $17, $2 + adc $18, $3 + lodsq + adc $11, %rax + stosq + lodsq + adc $12, %rax + stosq + lodsq + adc $13, %rax + stosq + lodsq + adc $14, %rax + stosq + setc %al + + ": "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&other_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); + ($name:ident, $n_words:expr, $self_expr: expr, $other: expr) => ( uint_overflowing_add_reg!($name, $n_words, $self_expr, $other) ) @@ -138,12 +184,13 @@ macro_rules! uint_overflowing_sub { let overflow: u8; unsafe { asm!(" - sbb $9, $0 - sbb $10, $1 - sbb $11, $2 - sbb $12, $3 - setb %al" - : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) + sub $9, $0 + sbb $10, $1 + sbb $11, $2 + sbb $12, $3 + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), "={al}"(overflow) : "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), "mr"(other_t[0]), "mr"(other_t[1]), "mr"(other_t[2]), "mr"(other_t[3]) : : @@ -151,6 +198,51 @@ macro_rules! uint_overflowing_sub { } (U256(result), overflow != 0) }); + (U512, $n_words: expr, $self_expr: expr, $other: expr) => ({ + let mut result: [u64; 8] = unsafe { mem::uninitialized() }; + let self_t: &[u64; 8] = unsafe { &mem::transmute($self_expr) }; + let other_t: &[u64; 8] = unsafe { &mem::transmute($other) }; + + let overflow: u8; + + unsafe { + asm!(" + sub $15, $0 + sbb $16, $1 + sbb $17, $2 + sbb $18, $3 + lodsq + sbb $19, %rax + stosq + lodsq + sbb $20, %rax + stosq + lodsq + sbb $21, %rax + stosq + lodsq + sbb $22, %rax + stosq + setb %al + " + : "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3]), + + "={al}"(overflow) /* $0 - $4 */ + + : "{rdi}"(&result[4] as *const u64) /* $5 */ + "{rsi}"(&self_t[4] as *const u64) /* $6 */ + "0"(self_t[0]), "1"(self_t[1]), "2"(self_t[2]), "3"(self_t[3]), + "m"(self_t[4]), "m"(self_t[5]), "m"(self_t[6]), "m"(self_t[7]), + /* $7 - $14 */ + + "m"(other_t[0]), "m"(other_t[1]), "m"(other_t[2]), "m"(other_t[3]), + "m"(other_t[4]), "m"(other_t[5]), "m"(other_t[6]), "m"(other_t[7]) /* $15 - $22 */ + : "rdi", "rsi" + : + ); + } + (U512(result), overflow != 0) + }); ($name:ident, $n_words: expr, $self_expr: expr, $other: expr) => ({ let res = overflowing!((!$other).overflowing_add(From::from(1u64))); let res = overflowing!($self_expr.overflowing_add(res)); @@ -251,8 +343,8 @@ macro_rules! uint_overflowing_mul { jrcxz 2f popcnt $7, %rcx - 2: - " + 2: + " : /* $0 */ "={r8}"(result[0]), /* $1 */ "={r9}"(result[1]), /* $2 */ "={r10}"(result[2]), /* $3 */ "={r11}"(result[3]), /* $4 */ "={rcx}"(overflow) @@ -1483,6 +1575,38 @@ mod tests { assert_eq!(format!("{}", U256::from(0)), "0"); } + #[test] + fn u512_multi_adds() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([1, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([1, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([2, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 2])); + + let (result, _) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 5, 2])); + + let (result, _) = U512([1, 2, 3, 4, 5, 6, 7, 8]).overflowing_add(U512([9, 10, 11, 12, 13, 14, 15, 16])); + assert_eq!(result, U512([10, 12, 14, 16, 18, 20, 22, 24])); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 2, 1]).overflowing_add(U512([0, 0, 0, 0, 0, 0, 3, 1])); + assert!(!overflow); + + let (_, overflow) = U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX]) + .overflowing_add(U512([::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX])); + assert!(overflow); + + let (_, overflow) = U512([0, 0, 0, 0, 0, 0, 0, ::std::u64::MAX]) + .overflowing_add(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert!(!overflow); + } #[test] fn u256_multi_adds() { @@ -1531,6 +1655,21 @@ mod tests { } + #[test] + fn u512_multi_subs() { + let (result, _) = U512([0, 0, 0, 0, 0, 0, 0, 0]).overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, 0])); + assert_eq!(result, U512([0, 0, 0, 0, 0, 0, 0, 0])); + + let (result, _) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert_eq!(result, U512([1, 1, 1, 1, 1, 1, 1, 1])); + + let (_, overflow) = U512([10, 9, 8, 7, 6, 5, 4, 3]).overflowing_sub(U512([9, 8, 7, 6, 5, 4, 3, 2])); + assert!(!overflow); + + let (_, overflow) = U512([9, 8, 7, 6, 5, 4, 3, 2]).overflowing_sub(U512([10, 9, 8, 7, 6, 5, 4, 3])); + assert!(overflow); + } + #[test] fn u256_multi_muls() { let (result, _) = U256([0, 0, 0, 0]).overflowing_mul(U256([0, 0, 0, 0])); From 98a75d06d16c2e2924ce4d3bc6f950f87e4753b4 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 26 Feb 2016 23:37:13 +0300 Subject: [PATCH 45/48] benches --- util/benches/bigint.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/util/benches/bigint.rs b/util/benches/bigint.rs index 3b2012bc7..da82084b8 100644 --- a/util/benches/bigint.rs +++ b/util/benches/bigint.rs @@ -38,7 +38,6 @@ fn u256_add(b: &mut Bencher) { }); } - #[bench] fn u256_sub(b: &mut Bencher) { b.iter(|| { @@ -47,6 +46,25 @@ fn u256_sub(b: &mut Bencher) { }); } +#[bench] +fn u512_sub(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([rand::random::(), rand::random::(), rand::random::(), rand::random::(), + rand::random::(), rand::random::(), rand::random::(), rand::random::()]), + |old, new| { old.overflowing_sub(U512([0, 0, 0, 0, 0, 0, 0, new])).0 }) + }); +} + +#[bench] +fn u512_add(b: &mut Bencher) { + b.iter(|| { + let n = black_box(10000); + (0..n).fold(U512([0, 0, 0, 0, 0, 0, 0, 0]), + |old, new| { old.overflowing_add(U512([new, new, new, new, new, new, new, new])).0 }) + }); +} + #[bench] fn u256_mul(b: &mut Bencher) { b.iter(|| { From 52faf8164d36d627bbdfde3f634ada216bf8e525 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Feb 2016 21:38:05 +0100 Subject: [PATCH 46/48] Update store.rs --- util/src/keys/store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 3b1ca4727..c4fa377f9 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -64,7 +64,8 @@ impl SecretStore { /// new instance of Secret Store in default home directory pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); - path.push("keystore"); + path.push(".parity"); + path.push("keys"); Self::new_in(&path) } From 5d5471f981a02915aa4be2626e63ed452364e40a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Feb 2016 22:40:32 +0100 Subject: [PATCH 47/48] Fix minor typo. --- parity/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index b50994cb9..c0c260371 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -71,8 +71,8 @@ Options: --listen-address URL Specify the IP/port on which to listen for peers [default: 0.0.0.0:30304]. --public-address URL Specify the IP/port on which peers may connect. --address URL Equivalent to --listen-address URL --public-address URL. - --peers NUM Try to maintain that many peers [default: 25]. - --no-discovery Disable new peer discovery. + --peers NUM Try to maintain that many peers [default: 25]. + --no-discovery Disable new peer discovery. --no-upnp Disable trying to figure out the correct public adderss over UPnP. --node-key KEY Specify node secret key, either as 64-character hex string or input to SHA3 operation. From 9b244022c7988da47674b380a903cd04e65b3498 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Feb 2016 22:44:33 +0100 Subject: [PATCH 48/48] Fix up README. --- README.md | 48 +++++++++++------------------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 575234622..f8b24f088 100644 --- a/README.md +++ b/README.md @@ -15,54 +15,28 @@ ### Building from source -##### Ubuntu 14.04, 15.04, 15.10 +First (if you don't already have it) get multirust: +- Linux: ```bash -# install multirust -curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release -``` - -##### Other Linux - -```bash -# install rust beta curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes - -# download and build parity -git clone https://github.com/ethcore/parity -cd parity - -# parity should be build with rust beta -multirust override beta - -# build in release -cargo build --release ``` -##### OSX with Homebrew +- OSX with Homebrew: +```bash +brew update && brew install multirust +``` + +Then, download and build Parity: ```bash -# install multirust -brew update -brew install multirust - -# download and build parity +# download Parity code git clone https://github.com/ethcore/parity cd parity -# use rust beta for building parity +# parity should be built with rust beta multirust override beta +# build in release mode cargo build --release ``` -