From 734652d9134127579928012fb9ec8fd4f55abb19 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Tue, 16 Feb 2016 00:56:25 +0300 Subject: [PATCH 01/19] work in progress --- util/src/keys/directory.rs | 46 +++++++++++++++++---------- util/src/keys/encryptor.rs | 65 ++++++++++++++++++++++++++++++++++++++ util/src/keys/mod.rs | 2 ++ util/src/lib.rs | 1 + 4 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 util/src/keys/encryptor.rs diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index c5e17100f..23e82d010 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -425,17 +425,17 @@ enum KeyFileLoadError { pub struct KeyDirectory { /// Directory path for key management. path: String, - cache: HashMap, - cache_usage: VecDeque, + cache: RefCell>, + cache_usage: RefCell>, } impl KeyDirectory { /// Initializes new cache directory context with a given `path` pub fn new(path: &Path) -> KeyDirectory { KeyDirectory { - cache: HashMap::new(), + cache: RefCell::new(HashMap::new()), path: path.to_str().expect("Initialized key directory with empty path").to_owned(), - cache_usage: VecDeque::new(), + cache_usage: RefCell::new(VecDeque::new()), } } @@ -448,25 +448,34 @@ impl KeyDirectory { let json_bytes = json_text.into_bytes(); try!(file.write(&json_bytes)); } + let mut cache = self.cache.borrow_mut(); let id = key_file.id.clone(); - self.cache.insert(id.clone(), key_file); + cache.insert(id.clone(), key_file); Ok(id.clone()) } /// Returns key given by id if corresponding file exists and no load error occured. /// Warns if any error occured during the key loading - pub fn get(&mut self, id: &Uuid) -> Option<&KeyFileContent> { + pub fn get(&self, id: &Uuid) -> Option> { let path = self.key_path(id); - self.cache_usage.push_back(id.clone()); - Some(self.cache.entry(id.to_owned()).or_insert( + { + let mut usage = self.cache_usage.borrow_mut(); + usage.push_back(id.clone()); + } + + if !self.cache.borrow().contains_key(id) { match KeyDirectory::load_key(&path) { - Ok(loaded_key) => loaded_key, + Ok(loaded_key) => { + self.cache.borrow_mut().insert(id.to_owned(), loaded_key); + } Err(error) => { warn!(target: "sstore", "error loading key {:?}: {:?}", id, error); return None; } } - )) + } + + Some(Ref::map(self.cache.borrow(), |c| c.get(id).expect("should be they key, we have just inserted or checked it"))) } /// Returns current path to the directory with keys @@ -476,29 +485,32 @@ impl KeyDirectory { /// Removes keys that never been requested during last `MAX_USAGE_TRACK` times pub fn collect_garbage(&mut self) { - let total_usages = self.cache_usage.len(); + let mut cache_usage = self.cache_usage.borrow_mut(); + + let total_usages = cache_usage.len(); let untracked_usages = max(total_usages as i64 - MAX_CACHE_USAGE_TRACK as i64, 0) as usize; if untracked_usages > 0 { - self.cache_usage.drain(..untracked_usages); + cache_usage.drain(..untracked_usages); } - if self.cache.len() <= MAX_CACHE_USAGE_TRACK { return; } + let mut cache = self.cache.borrow_mut(); + if cache.len() <= MAX_CACHE_USAGE_TRACK { return; } - let uniqs: HashSet<&Uuid> = self.cache_usage.iter().collect(); + let uniqs: HashSet<&Uuid> = cache_usage.iter().collect(); let mut removes = HashSet::new(); - for key in self.cache.keys() { + for key in cache.keys() { if !uniqs.contains(key) { removes.insert(key.clone()); } } - for removed_key in removes { self.cache.remove(&removed_key); } + for removed_key in removes { cache.remove(&removed_key); } } /// Reports how many keys are currently cached. pub fn cache_size(&self) -> usize { - self.cache.len() + self.cache.borrow().len() } fn key_path(&self, id: &Uuid) -> PathBuf { diff --git a/util/src/keys/encryptor.rs b/util/src/keys/encryptor.rs new file mode 100644 index 000000000..cbc926dc6 --- /dev/null +++ b/util/src/keys/encryptor.rs @@ -0,0 +1,65 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Generic Encryptor + +use keys::directory::*; +use common::*; + +pub trait EncryptedHashMap { + // Returns existing value for the key, if any + fn get(&self, key: &Key, password: &str) -> Option; + // Insert new encrypted key-value and returns previous if there was any + fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; + // Removes key-value by key and returns the removed one, if any exists and password was provided + fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; + // Deletes key-value by key and returns if the key-value existed + fn delete(&mut self, key: &Key) -> bool { + self.remove::<()>(key, None).is_some() + } +} + +pub struct SecretStore { + directory: KeyDirectory +} + +impl EncryptedHashMap for SecretStore { + fn get(&self, key: &H128, password: &str) -> Option { + match self.directory.get(key) { + Some(key_file) => { + let mut instance = Value::default(); + instance.populate_raw(&key_file.crypto.cipher_text); + Some(instance) + }, + None => None + } + + } + + fn insert(&mut self, key: H128, value: Value, password: &str) -> Option{ + let previous = if let Some(key_file) = self.directory.get(&key) { self.get(&key, password) } else { None }; + previous + } + + fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { + let previous = match (self.directory.get(&key), password) { + (Some(key_file), Some(pass)) => self.get(&key, pass), + (_, _) => None + }; + previous + } + +} diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index d7ffdb0dd..f886d362c 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -17,3 +17,5 @@ //! Key management module pub mod directory; + +mod encryptor; diff --git a/util/src/lib.rs b/util/src/lib.rs index d4f972800..151afb60a 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -20,6 +20,7 @@ #![feature(associated_consts)] #![feature(plugin)] #![feature(catch_panic)] +#![feature(cell_extras)] // Clippy settings #![plugin(clippy)] // TODO [todr] not really sure From 26e992ad2ed4381762f15c3544b63eae951d40b8 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Tue, 16 Feb 2016 11:30:22 +0300 Subject: [PATCH 02/19] encrypting any bytes covertible now via hashmap --- util/src/bytes.rs | 4 +++ util/src/keys/directory.rs | 26 +++++++------- util/src/keys/encryptor.rs | 74 +++++++++++++++++++++++++++++++++----- 3 files changed, 82 insertions(+), 22 deletions(-) diff --git a/util/src/bytes.rs b/util/src/bytes.rs index 5ad2660e8..0006827e8 100644 --- a/util/src/bytes.rs +++ b/util/src/bytes.rs @@ -178,6 +178,10 @@ impl BytesConvertable for Vec { fn bytes(&self) -> &[u8] { self } } +impl BytesConvertable for String { + fn bytes(&self) -> &[u8] { &self.as_bytes() } +} + macro_rules! impl_bytes_convertable_for_array { ($zero: expr) => (); ($len: expr, $($idx: expr),*) => { diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 23e82d010..ccd7d8fb4 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -27,7 +27,7 @@ const MAX_CACHE_USAGE_TRACK: usize = 128; #[derive(PartialEq, Debug, Clone)] pub enum CryptoCipherType { /// aes-128-ctr with 128-bit initialisation vector(iv) - Aes128Ctr(U128) + Aes128Ctr(H128) } #[derive(PartialEq, Debug, Clone)] @@ -182,7 +182,7 @@ impl KeyFileCrypto { Some("aes-128-ctr") => CryptoCipherType::Aes128Ctr( match try!(as_object.get("cipherparams").ok_or(CryptoParseError::NoCipherParameters)).as_object() { None => { return Err(CryptoParseError::NoCipherParameters); }, - Some(cipher_param) => match U128::from_str(match cipher_param["iv"].as_string() { + Some(cipher_param) => match H128::from_str(match cipher_param["iv"].as_string() { None => { return Err(CryptoParseError::NoInitialVector); }, Some(iv_hex_string) => iv_hex_string }) @@ -231,7 +231,7 @@ impl KeyFileCrypto { fn to_json(&self) -> Json { let mut map = BTreeMap::new(); match self.cipher_type { - CryptoCipherType::Aes128Ctr(iv) => { + CryptoCipherType::Aes128Ctr(ref iv) => { map.insert("cipher".to_owned(), Json::String("aes-128-ctr".to_owned())); let mut cipher_params = BTreeMap::new(); cipher_params.insert("iv".to_owned(), Json::String(format!("{:?}", iv))); @@ -260,7 +260,7 @@ impl KeyFileCrypto { /// `c` - number of iterations for derived key. /// `salt` - cryptographic site, random 256-bit hash (ensure it's crypto-random). /// `iv` - initialisation vector. - pub fn new_pbkdf2(cipher_text: Bytes, iv: U128, salt: H256, c: u32, dk_len: u32) -> KeyFileCrypto { + pub fn new_pbkdf2(cipher_text: Bytes, iv: H128, salt: H256, c: u32, dk_len: u32) -> KeyFileCrypto { KeyFileCrypto { cipher_type: CryptoCipherType::Aes128Ctr(iv), cipher_text: cipher_text, @@ -331,7 +331,7 @@ enum CryptoParseError { InvalidCipherType(Mismatch), NoInitialVector, NoCipherParameters, - InvalidInitialVector(FromHexError), + InvalidInitialVector(UtilError), NoKdf, NoKdfType, Scrypt(ScryptParseError), @@ -832,14 +832,14 @@ mod file_tests { #[test] fn can_create_key_with_new_id() { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32)); assert!(!uuid_to_string(&key.id).is_empty()); } #[test] fn can_load_json_from_itself() { let cipher_text: Bytes = FromHex::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaa22222222222222222222222").unwrap(); - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32)); let json = key.to_json(); let loaded_key = KeyFileContent::from_json(&json).unwrap(); @@ -997,7 +997,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32))).unwrap(); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32))).unwrap(); let path = directory.key_path(&uuid); let key = KeyDirectory::load_key(&path).unwrap(); @@ -1013,7 +1013,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..1000 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } @@ -1033,7 +1033,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..1000 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } @@ -1066,7 +1066,7 @@ mod specs { let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, U128::zero(), H256::random(), 32, 32))); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32))); assert!(uuid.is_ok()); } @@ -1076,7 +1076,7 @@ mod specs { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32))).unwrap(); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32))).unwrap(); let key = directory.get(&uuid).unwrap(); @@ -1091,7 +1091,7 @@ mod specs { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..10 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), U128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } diff --git a/util/src/keys/encryptor.rs b/util/src/keys/encryptor.rs index cbc926dc6..7645a2b84 100644 --- a/util/src/keys/encryptor.rs +++ b/util/src/keys/encryptor.rs @@ -18,17 +18,25 @@ use keys::directory::*; use common::*; +use rcrypto::pbkdf2::*; +use rcrypto::aes; +use rcrypto::hmac::*; +use crypto; + +const KEY_LENGTH: u32 = 32; +const KEY_ITERATIONS: u32 = 4096; +const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; pub trait EncryptedHashMap { // Returns existing value for the key, if any - fn get(&self, key: &Key, password: &str) -> Option; + fn get(&self, key: &Key, password: &str) -> Option; // Insert new encrypted key-value and returns previous if there was any - fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; + fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; // Removes key-value by key and returns the removed one, if any exists and password was provided - fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; + fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; // Deletes key-value by key and returns if the key-value existed fn delete(&mut self, key: &Key) -> bool { - self.remove::<()>(key, None).is_some() + self.remove::<&[u8]>(key, None).is_some() } } @@ -36,8 +44,25 @@ pub struct SecretStore { directory: KeyDirectory } +impl SecretStore { + fn new() -> SecretStore { + let mut path = ::std::env::home_dir().expect("Failed to get home dir"); + path.push(".keys"); + SecretStore { + directory: KeyDirectory::new(&path) + } + } + + #[cfg(test)] + fn new_test(path: &::tests::helpers::RandomTempPath) -> SecretStore { + SecretStore { + directory: KeyDirectory::new(path.as_path()) + } + } +} + impl EncryptedHashMap for SecretStore { - fn get(&self, key: &H128, password: &str) -> Option { + fn get(&self, key: &H128, password: &str) -> Option { match self.directory.get(key) { Some(key_file) => { let mut instance = Value::default(); @@ -49,17 +74,48 @@ impl EncryptedHashMap for SecretStore { } - fn insert(&mut self, key: H128, value: Value, password: &str) -> Option{ - let previous = if let Some(key_file) = self.directory.get(&key) { self.get(&key, password) } else { None }; + fn insert(&mut self, key: H128, value: Value, password: &str) -> Option { + let previous = if let Some(_) = self.directory.get(&key) { self.get(&key, password) } else { None }; + + let salt = H256::random(); + let iv = H128::random(); + let mut key_file = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(vec![], iv.clone(), salt.clone(), KEY_ITERATIONS, KEY_LENGTH)); + + let mut mac = Hmac::new(::rcrypto::sha2::Sha256::new(), password.as_bytes()); + let mut derived_key = vec![0u8; KEY_LENGTH as usize]; + pbkdf2(&mut mac, &salt.as_slice(), KEY_ITERATIONS, &mut derived_key); + let key = &derived_key[KEY_LENGTH_AES as usize..KEY_LENGTH as usize]; + + let mut cipher_text = vec![0u8; value.as_slice().len()]; + crypto::aes::encrypt(&key, &iv.as_slice(), &value.as_slice(), &mut cipher_text); + key_file.crypto.cipher_text = cipher_text; + previous } - fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { + fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { let previous = match (self.directory.get(&key), password) { - (Some(key_file), Some(pass)) => self.get(&key, pass), + (Some(_), Some(pass)) => self.get(&key, pass), (_, _) => None }; previous } } + + +#[cfg(test)] +mod tests { + use super::*; + use tests::helpers::*; + use common::*; + + #[test] + fn secret_store_insert() { + let temp = RandomTempPath::create_dir(); + let mut sstore = SecretStore::new_test(&temp); + + sstore.insert(H128::random(), "Cat".to_owned(), "pass"); + } + +} From a649d6f1311b75e8328e840537502fdd4dc73c61 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Tue, 16 Feb 2016 19:19:32 +0300 Subject: [PATCH 03/19] first vector up --- util/src/keys/directory.rs | 21 +++++++- util/src/keys/mod.rs | 2 +- util/src/keys/{encryptor.rs => store.rs} | 61 +++++++++++++++++++++--- 3 files changed, 75 insertions(+), 9 deletions(-) rename util/src/keys/{encryptor.rs => store.rs} (60%) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index ccd7d8fb4..641a5cae4 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -168,6 +168,8 @@ pub struct KeyFileCrypto { pub cipher_text: Bytes, /// Password derived key generator function settings. pub kdf: KeyFileKdf, + /// Mac + pub mac: H256 } impl KeyFileCrypto { @@ -216,15 +218,24 @@ impl KeyFileCrypto { } }; - let cipher_text = match as_object["ciphertext"].as_string() { - None => { return Err(CryptoParseError::NoCipherText); } + let cipher_text = match try!(as_object.get("ciphertext").ok_or(CryptoParseError::NoCipherText)).as_string() { + None => { return Err(CryptoParseError::InvalidCipherText); } Some(text) => text }; + let mac: H256 = match try!(as_object.get("mac").ok_or(CryptoParseError::NoMac)).as_string() { + None => { return Err(CryptoParseError::InvalidMacFormat(None)) }, + Some(salt_value) => match H256::from_str(salt_value) { + Ok(salt_hex_value) => salt_hex_value, + Err(from_hex_error) => { return Err(CryptoParseError::InvalidMacFormat(Some(from_hex_error))); }, + } + }; + Ok(KeyFileCrypto { cipher_text: Bytes::from(cipher_text), cipher_type: cipher_type, kdf: kdf, + mac: mac, }) } @@ -251,6 +262,8 @@ impl KeyFileCrypto { KeyFileKdf::Scrypt(ref scrypt_params) => scrypt_params.to_json() }); + map.insert("mac".to_owned(), Json::String(format!("{:?}", self.mac))); + Json::Object(map) } @@ -270,6 +283,7 @@ impl KeyFileCrypto { c: c, prf: Pbkdf2CryptoFunction::HMacSha256 }), + mac: H256::random(), } } } @@ -324,7 +338,10 @@ pub struct KeyFileContent { #[derive(Debug)] enum CryptoParseError { + InvalidMacFormat(Option), + NoMac, NoCipherText, + InvalidCipherText, NoCipherType, InvalidJsonFormat, InvalidKdfType(Mismatch), diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index f886d362c..abd029444 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -18,4 +18,4 @@ pub mod directory; -mod encryptor; +mod store; diff --git a/util/src/keys/encryptor.rs b/util/src/keys/store.rs similarity index 60% rename from util/src/keys/encryptor.rs rename to util/src/keys/store.rs index 7645a2b84..9972b6e57 100644 --- a/util/src/keys/encryptor.rs +++ b/util/src/keys/store.rs @@ -27,6 +27,9 @@ const KEY_LENGTH: u32 = 32; const KEY_ITERATIONS: u32 = 4096; const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; +const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize; +const KEY_LENGTH_AES_USIZE: usize = KEY_LENGTH_AES as usize; + pub trait EncryptedHashMap { // Returns existing value for the key, if any fn get(&self, key: &Key, password: &str) -> Option; @@ -61,6 +64,26 @@ impl SecretStore { } } +fn derive_key_iterations(password: &str, salt: &H256, c: u32) -> (Bytes, Bytes) { + let mut h_mac = Hmac::new(::rcrypto::sha2::Sha256::new(), password.as_bytes()); + let mut derived_key = vec![0u8; KEY_LENGTH_USIZE]; + pbkdf2(&mut h_mac, &salt.as_slice(), c, &mut derived_key); + let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE]; + let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE]; + (derived_right_bits.to_vec(), derived_left_bits.to_vec()) +} + +fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) { + derive_key_iterations(password, salt, KEY_ITERATIONS) +} + +fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { + let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()]; + mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits); + mac[KEY_LENGTH_AES_USIZE..cipher_text.len()+KEY_LENGTH_AES_USIZE].clone_from_slice(cipher_text); + mac +} + impl EncryptedHashMap for SecretStore { fn get(&self, key: &H128, password: &str) -> Option { match self.directory.get(key) { @@ -81,14 +104,13 @@ impl EncryptedHashMap for SecretStore { let iv = H128::random(); let mut key_file = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(vec![], iv.clone(), salt.clone(), KEY_ITERATIONS, KEY_LENGTH)); - let mut mac = Hmac::new(::rcrypto::sha2::Sha256::new(), password.as_bytes()); - let mut derived_key = vec![0u8; KEY_LENGTH as usize]; - pbkdf2(&mut mac, &salt.as_slice(), KEY_ITERATIONS, &mut derived_key); - let key = &derived_key[KEY_LENGTH_AES as usize..KEY_LENGTH as usize]; + let (derived_left_bits, derived_right_bits) = derive_key(password, &salt); let mut cipher_text = vec![0u8; value.as_slice().len()]; - crypto::aes::encrypt(&key, &iv.as_slice(), &value.as_slice(), &mut cipher_text); - key_file.crypto.cipher_text = cipher_text; + crypto::aes::encrypt(&derived_left_bits, &iv.as_slice(), &value.as_slice(), &mut cipher_text); + key_file.crypto.cipher_text = cipher_text.clone(); + + key_file.crypto.mac = derive_mac(&derived_right_bits, &cipher_text).sha3(); previous } @@ -103,6 +125,31 @@ impl EncryptedHashMap for SecretStore { } +#[cfg(test)] +mod vector_tests { + use super::{derive_key,derive_mac,derive_key_iterations}; + use common::*; + + + #[test] + fn mac_vector() { + let password = "testpassword"; + let salt = H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(); + let cipher_text = FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(); + let iterations = 262144u32; + + let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, &salt, iterations); + assert_eq!("f06d69cdc7da0faffb1008270bca38f5", derived_left_bits.to_hex()); + assert_eq!("e31891a3a773950e6d0fea48a7188551", derived_right_bits.to_hex()); + + let mut mac_body = derive_mac(&derived_right_bits, &cipher_text); + assert_eq!("e31891a3a773950e6d0fea48a71885515318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46", mac_body.to_hex()); + + let mac = mac_body.sha3(); + assert_eq!("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2", format!("{:?}", mac)); + } +} + #[cfg(test)] mod tests { @@ -118,4 +165,6 @@ mod tests { sstore.insert(H128::random(), "Cat".to_owned(), "pass"); } + + } From 9895f00e5ebebd8a4d42a46422ffed5d946e282c Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Tue, 16 Feb 2016 20:52:36 +0300 Subject: [PATCH 04/19] warnings, docs, and finding bugs --- util/src/keys/directory.rs | 36 +++++++--- util/src/keys/mod.rs | 3 +- util/src/keys/store.rs | 142 ++++++++++++++++++++++++++++++------- 3 files changed, 143 insertions(+), 38 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 641a5cae4..7656a938f 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -273,7 +273,7 @@ impl KeyFileCrypto { /// `c` - number of iterations for derived key. /// `salt` - cryptographic site, random 256-bit hash (ensure it's crypto-random). /// `iv` - initialisation vector. - pub fn new_pbkdf2(cipher_text: Bytes, iv: H128, salt: H256, c: u32, dk_len: u32) -> KeyFileCrypto { + pub fn new_pbkdf2(cipher_text: Bytes, iv: H128, salt: H256, mac: H256, c: u32, dk_len: u32) -> KeyFileCrypto { KeyFileCrypto { cipher_type: CryptoCipherType::Aes128Ctr(iv), cipher_text: cipher_text, @@ -283,7 +283,7 @@ impl KeyFileCrypto { c: c, prf: Pbkdf2CryptoFunction::HMacSha256 }), - mac: H256::random(), + mac: mac, } } } @@ -530,6 +530,22 @@ impl KeyDirectory { self.cache.borrow().len() } + /// Removes key file from key directory + pub fn delete(&mut self, id: &Uuid) -> Result<(), ::std::io::Error> { + let path = self.key_path(id); + + if !self.cache.borrow().contains_key(id) { + return match fs::remove_file(&path) { + Ok(_) => { + self.cache.borrow_mut().remove(&id); + Ok(()) + }, + Err(e) => Err(e) + }; + } + Ok(()) + } + fn key_path(&self, id: &Uuid) -> PathBuf { let mut path = PathBuf::new(); path.push(self.path.clone()); @@ -849,14 +865,14 @@ mod file_tests { #[test] fn can_create_key_with_new_id() { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32)); assert!(!uuid_to_string(&key.id).is_empty()); } #[test] fn can_load_json_from_itself() { let cipher_text: Bytes = FromHex::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaa22222222222222222222222").unwrap(); - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32)); let json = key.to_json(); let loaded_key = KeyFileContent::from_json(&json).unwrap(); @@ -1014,7 +1030,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32))).unwrap(); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32))).unwrap(); let path = directory.key_path(&uuid); let key = KeyDirectory::load_key(&path).unwrap(); @@ -1030,7 +1046,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..1000 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } @@ -1050,7 +1066,7 @@ mod directory_tests { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..1000 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } @@ -1083,7 +1099,7 @@ mod specs { let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), 32, 32))); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text, H128::zero(), H256::random(), H256::random(), 32, 32))); assert!(uuid.is_ok()); } @@ -1093,7 +1109,7 @@ mod specs { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let temp_path = RandomTempPath::create_dir(); let mut directory = KeyDirectory::new(&temp_path.as_path()); - let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32))).unwrap(); + let uuid = directory.save(KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32))).unwrap(); let key = directory.get(&uuid).unwrap(); @@ -1108,7 +1124,7 @@ mod specs { let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); let mut keys = Vec::new(); for _ in 0..10 { - let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), 32, 32)); + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32)); keys.push(directory.save(key).unwrap()); } diff --git a/util/src/keys/mod.rs b/util/src/keys/mod.rs index abd029444..fd52136d7 100644 --- a/util/src/keys/mod.rs +++ b/util/src/keys/mod.rs @@ -17,5 +17,4 @@ //! Key management module pub mod directory; - -mod store; +pub mod store; diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index 9972b6e57..f473d0ea2 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -14,12 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Generic Encryptor +//! Secret Store use keys::directory::*; use common::*; use rcrypto::pbkdf2::*; -use rcrypto::aes; use rcrypto::hmac::*; use crypto; @@ -30,25 +29,37 @@ const KEY_LENGTH_AES: u32 = KEY_LENGTH/2; const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize; const KEY_LENGTH_AES_USIZE: usize = KEY_LENGTH_AES as usize; +/// Encrypted hash-map, each request should contain password pub trait EncryptedHashMap { - // Returns existing value for the key, if any - fn get(&self, key: &Key, password: &str) -> Option; - // Insert new encrypted key-value and returns previous if there was any + /// Returns existing value for the key, if any + fn get(&self, key: &Key, password: &str) -> Result; + /// Insert new encrypted key-value and returns previous if there was any fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; - // Removes key-value by key and returns the removed one, if any exists and password was provided + /// Removes key-value by key and returns the removed one, if any exists and password was provided fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; - // Deletes key-value by key and returns if the key-value existed + /// Deletes key-value by key and returns if the key-value existed fn delete(&mut self, key: &Key) -> bool { self.remove::<&[u8]>(key, None).is_some() } } +/// Error retrieving value from encrypted hashmap +#[derive(Debug)] +pub enum EncryptedHashMapError { + /// Encryption failed + InvalidPassword, + /// No key in the hashmap + UnknownIdentifier +} + +/// Represent service for storing encrypted arbitrary data pub struct SecretStore { directory: KeyDirectory } impl SecretStore { - fn new() -> SecretStore { + /// new instance of Secret Store + pub fn new() -> SecretStore { let mut path = ::std::env::home_dir().expect("Failed to get home dir"); path.push(".keys"); SecretStore { @@ -85,41 +96,75 @@ fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { } impl EncryptedHashMap for SecretStore { - fn get(&self, key: &H128, password: &str) -> Option { + fn get(&self, key: &H128, password: &str) -> Result { match self.directory.get(key) { Some(key_file) => { - let mut instance = Value::default(); - instance.populate_raw(&key_file.crypto.cipher_text); - Some(instance) - }, - None => None - } + let decrypted_bytes = match key_file.crypto.kdf { + KeyFileKdf::Pbkdf2(ref params) => { + let (derived_left_bits, derived_right_bits) = derive_key(password, ¶ms.salt); + let expected_mac = derive_mac(&derived_right_bits, &key_file.crypto.cipher_text).sha3(); + if expected_mac != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } + let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; + match key_file.crypto.cipher_type { + CryptoCipherType::Aes128Ctr(ref iv) => { + crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val); + } + } + val + } + _ => { unimplemented!(); } + }; + + let mut instance = Value::default(); + instance.populate_raw(&decrypted_bytes); + Ok(instance) + }, + None => Err(EncryptedHashMapError::UnknownIdentifier) + } } fn insert(&mut self, key: H128, value: Value, password: &str) -> Option { - let previous = if let Some(_) = self.directory.get(&key) { self.get(&key, password) } else { None }; + let previous = if let Ok(previous_value) = self.get(&key, password) { Some(previous_value) } else { None }; + // crypto random initiators let salt = H256::random(); let iv = H128::random(); - let mut key_file = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(vec![], iv.clone(), salt.clone(), KEY_ITERATIONS, KEY_LENGTH)); + // two parts of derived key + // DK = [ DK[0..15] DK[16..31] ] = [derived_left_bits, derived_right_bits] let (derived_left_bits, derived_right_bits) = derive_key(password, &salt); let mut cipher_text = vec![0u8; value.as_slice().len()]; - crypto::aes::encrypt(&derived_left_bits, &iv.as_slice(), &value.as_slice(), &mut cipher_text); - key_file.crypto.cipher_text = cipher_text.clone(); + // aes-128-ctr with initial vector of iv + crypto::aes::encrypt(&derived_left_bits, &iv.clone(), &value.as_slice(), &mut cipher_text); - key_file.crypto.mac = derive_mac(&derived_right_bits, &cipher_text).sha3(); + // KECCAK(DK[16..31] ++ ), where DK[16..31] - derived_right_bits + let mac = derive_mac(&derived_right_bits, &cipher_text.clone()).sha3(); + let key_file = KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + cipher_text, + iv, + salt, + mac, + KEY_ITERATIONS, + KEY_LENGTH)); + if let Err(io_error) = self.directory.save(key_file) { + warn!("Error saving key file: {:?}", io_error); + } previous } fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { - let previous = match (self.directory.get(&key), password) { - (Some(_), Some(pass)) => self.get(&key, pass), - (_, _) => None - }; + let previous = if let Some(pass) = password { + if let Ok(previous_value) = self.get(&key, pass) { Some(previous_value) } else { None } + } + else { None }; + + if let Err(io_error) = self.directory.delete(key) { + warn!("Error saving key file: {:?}", io_error); + } previous } @@ -127,7 +172,7 @@ impl EncryptedHashMap for SecretStore { #[cfg(test)] mod vector_tests { - use super::{derive_key,derive_mac,derive_key_iterations}; + use super::{derive_mac,derive_key_iterations}; use common::*; @@ -142,7 +187,7 @@ mod vector_tests { assert_eq!("f06d69cdc7da0faffb1008270bca38f5", derived_left_bits.to_hex()); assert_eq!("e31891a3a773950e6d0fea48a7188551", derived_right_bits.to_hex()); - let mut mac_body = derive_mac(&derived_right_bits, &cipher_text); + let mac_body = derive_mac(&derived_right_bits, &cipher_text); assert_eq!("e31891a3a773950e6d0fea48a71885515318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46", mac_body.to_hex()); let mac = mac_body.sha3(); @@ -165,6 +210,51 @@ mod tests { sstore.insert(H128::random(), "Cat".to_owned(), "pass"); } + #[test] + fn secret_store_get_fail() { + let temp = RandomTempPath::create_dir(); + { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + write_sstore.directory.save( + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32))) + .unwrap(); + } + let sstore = SecretStore::new_test(&temp); + if let Ok(_) = sstore.get::(&H128::from_str("3198bc9c66725ab3d9954942343ae5b6").unwrap(), "testpassword") { + panic!("shoud be error loading key, we requested the wrong key"); + } + } + + #[test] + fn secret_store_get() { + let temp = RandomTempPath::create_dir(); + let key_id = { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + write_sstore.directory.save( + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32))) + .unwrap() + }; + let sstore = SecretStore::new_test(&temp); + if let Err(e) = sstore.get::(&key_id, "testpassword") { + panic!("got no key: {:?}", e); + } + } } From 63f6ab4e6d70d8a2e3a148a76dec82decd500326 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 02:32:16 +0300 Subject: [PATCH 05/19] resolved with bytes-encodable issue --- util/src/bytes.rs | 43 ++++++++++++++++++++++++++++++++++++++ util/src/keys/directory.rs | 2 +- util/src/keys/store.rs | 35 ++++++++++++++++++------------- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/util/src/bytes.rs b/util/src/bytes.rs index 0006827e8..a26804c5b 100644 --- a/util/src/bytes.rs +++ b/util/src/bytes.rs @@ -256,6 +256,49 @@ impl Populatable for [T] where T: Sized { } } +#[derive(Debug)] +/// Bytes array deserialization error +pub enum FromBytesError { + /// Not enough bytes for the requested type + NotLongEnough, + /// Too many bytes for the requested type + TooLong, +} + +/// Value that can be serialized from bytes array +pub trait FromRawBytes : Sized { + /// function that will instantiate and initialize object from slice + fn from_bytes(d: &[u8]) -> Result; +} + +impl FromRawBytes for T where T: Sized + FixedHash { + fn from_bytes(bytes: &[u8]) -> Result { + use std::mem; + use std::cmp::Ordering; + match bytes.len().cmp(&mem::size_of::()) { + Ordering::Less => return Err(FromBytesError::NotLongEnough), + Ordering::Greater => return Err(FromBytesError::TooLong), + Ordering::Equal => () + }; + + let mut res: Self = unsafe { mem::uninitialized() }; + res.copy_raw(bytes); + Ok(res) + } +} + +impl FromRawBytes for String { + fn from_bytes(bytes: &[u8]) -> Result { + Ok(::std::str::from_utf8(bytes).unwrap().to_owned()) + } +} + +impl FromRawBytes for Vec { + fn from_bytes(bytes: &[u8]) -> Result, FromBytesError> { + Ok(bytes.clone().to_vec()) + } +} + #[test] fn fax_raw() { let mut x = [255u8; 4]; diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 7656a938f..73f2542f4 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -232,7 +232,7 @@ impl KeyFileCrypto { }; Ok(KeyFileCrypto { - cipher_text: Bytes::from(cipher_text), + cipher_text: match FromHex::from_hex(cipher_text) { Ok(bytes) => bytes, Err(_) => { return Err(CryptoParseError::InvalidCipherText); } }, cipher_type: cipher_type, kdf: kdf, mac: mac, diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index f473d0ea2..c72a0a7f3 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -32,14 +32,14 @@ const KEY_LENGTH_AES_USIZE: usize = KEY_LENGTH_AES as usize; /// Encrypted hash-map, each request should contain password pub trait EncryptedHashMap { /// Returns existing value for the key, if any - fn get(&self, key: &Key, password: &str) -> Result; + fn get(&self, key: &Key, password: &str) -> Result; /// Insert new encrypted key-value and returns previous if there was any - fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; + fn insert(&mut self, key: Key, value: Value, password: &str) -> Option; /// Removes key-value by key and returns the removed one, if any exists and password was provided - fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; + fn remove (&mut self, key: &Key, password: Option<&str>) -> Option; /// Deletes key-value by key and returns if the key-value existed fn delete(&mut self, key: &Key) -> bool { - self.remove::<&[u8]>(key, None).is_some() + self.remove::(key, None).is_some() } } @@ -49,7 +49,9 @@ pub enum EncryptedHashMapError { /// Encryption failed InvalidPassword, /// No key in the hashmap - UnknownIdentifier + UnknownIdentifier, + /// Stored value is not well formed for the requested type + InvalidValueFormat(FromBytesError), } /// Represent service for storing encrypted arbitrary data @@ -96,14 +98,16 @@ fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes { } impl EncryptedHashMap for SecretStore { - fn get(&self, key: &H128, password: &str) -> Result { + fn get(&self, key: &H128, password: &str) -> Result { match self.directory.get(key) { Some(key_file) => { let decrypted_bytes = match key_file.crypto.kdf { KeyFileKdf::Pbkdf2(ref params) => { - let (derived_left_bits, derived_right_bits) = derive_key(password, ¶ms.salt); - let expected_mac = derive_mac(&derived_right_bits, &key_file.crypto.cipher_text).sha3(); - if expected_mac != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } + let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, ¶ms.salt, params.c); + //assert_eq!(derive_mac(&derived_right_bits, &key_file.crypto.cipher_text).to_hex(), ""); + assert_eq!(&key_file.crypto.cipher_text.to_hex(), "5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46"); + if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) + .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } let mut val = vec![0u8; key_file.crypto.cipher_text.len()]; match key_file.crypto.cipher_type { @@ -116,15 +120,16 @@ impl EncryptedHashMap for SecretStore { _ => { unimplemented!(); } }; - let mut instance = Value::default(); - instance.populate_raw(&decrypted_bytes); - Ok(instance) + match Value::from_bytes(&decrypted_bytes) { + Ok(value) => Ok(value), + Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error)) + } }, None => Err(EncryptedHashMapError::UnknownIdentifier) } } - fn insert(&mut self, key: H128, value: Value, password: &str) -> Option { + fn insert(&mut self, key: H128, value: Value, password: &str) -> Option { let previous = if let Ok(previous_value) = self.get(&key, password) { Some(previous_value) } else { None }; // crypto random initiators @@ -156,7 +161,7 @@ impl EncryptedHashMap for SecretStore { previous } - fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { + fn remove(&mut self, key: &H128, password: Option<&str>) -> Option { let previous = if let Some(pass) = password { if let Ok(previous_value) = self.get(&key, pass) { Some(previous_value) } else { None } } @@ -229,7 +234,7 @@ mod tests { } let sstore = SecretStore::new_test(&temp); if let Ok(_) = sstore.get::(&H128::from_str("3198bc9c66725ab3d9954942343ae5b6").unwrap(), "testpassword") { - panic!("shoud be error loading key, we requested the wrong key"); + panic!("should be error loading key, we requested the wrong key"); } } From 4a028f5faf220812f9e4138c0219a5fd5c8018dc Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 02:49:13 +0300 Subject: [PATCH 06/19] cleanup asserts --- util/src/keys/store.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index c72a0a7f3..b9ee57694 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -104,8 +104,6 @@ impl EncryptedHashMap for SecretStore { let decrypted_bytes = match key_file.crypto.kdf { KeyFileKdf::Pbkdf2(ref params) => { let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, ¶ms.salt, params.c); - //assert_eq!(derive_mac(&derived_right_bits, &key_file.crypto.cipher_text).to_hex(), ""); - assert_eq!(&key_file.crypto.cipher_text.to_hex(), "5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46"); if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text) .sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); } From d63f13245f38de879308c1e526114ee9f8abd53d Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 03:17:02 +0300 Subject: [PATCH 07/19] doc typo --- util/src/semantic_version.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/src/semantic_version.rs b/util/src/semantic_version.rs index 496ff2318..b550cd829 100644 --- a/util/src/semantic_version.rs +++ b/util/src/semantic_version.rs @@ -16,13 +16,13 @@ //! Semantic version formatting and comparing. -/// A version value with strict meaning. Use `to_u32` to convert to a simple integer. -/// +/// A version value with strict meaning. Use `as_u32` to convert to a simple integer. +/// /// # Example /// ``` /// extern crate ethcore_util as util; /// use util::semantic_version::*; -/// +/// /// fn main() { /// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203); /// } From 14b02ff26faf31069e0d6fa807abe1b7fb913934 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 11:48:12 +0300 Subject: [PATCH 08/19] tests and fixes --- util/src/keys/directory.rs | 33 +++++++++++++++++++++++++++++++++ util/src/keys/store.rs | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 73f2542f4..7cc101da1 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -546,6 +546,24 @@ impl KeyDirectory { Ok(()) } + /// Enumerates all keys in the directory + pub fn list(&self) -> Result, ::std::io::Error> { + let mut result = Vec::new(); + for entry in try!(fs::read_dir(&self.path)) { + let entry = try!(entry); + if !try!(fs::metadata(entry.path())).is_dir() { + match entry.file_name().to_str() { + Some(ref name) => { + if let Ok(uuid) = uuid_from_string(name) { result.push(uuid); } + }, + None => { continue; } + }; + + } + } + Ok(result) + } + fn key_path(&self, id: &Uuid) -> PathBuf { let mut path = PathBuf::new(); path.push(self.path.clone()); @@ -1130,4 +1148,19 @@ mod specs { assert_eq!(10, keys.len()) } + + #[test] + fn can_list_keys() { + let temp_path = RandomTempPath::create_dir(); + let mut directory = KeyDirectory::new(&temp_path.as_path()); + + let cipher_text: Bytes = FromHex::from_hex("a0f05555").unwrap(); + let mut keys = Vec::new(); + for _ in 0..33 { + let key = KeyFileContent::new(KeyFileCrypto::new_pbkdf2(cipher_text.clone(), H128::zero(), H256::random(), H256::random(), 32, 32)); + keys.push(directory.save(key).unwrap()); + } + + assert_eq!(33, directory.list().unwrap().len()); + } } diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index b9ee57694..e122ce808 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -145,7 +145,7 @@ impl EncryptedHashMap for SecretStore { // KECCAK(DK[16..31] ++ ), where DK[16..31] - derived_right_bits let mac = derive_mac(&derived_right_bits, &cipher_text.clone()).sha3(); - let key_file = KeyFileContent::new( + let mut key_file = KeyFileContent::new( KeyFileCrypto::new_pbkdf2( cipher_text, iv, @@ -153,6 +153,7 @@ impl EncryptedHashMap for SecretStore { mac, KEY_ITERATIONS, KEY_LENGTH)); + key_file.id = key; if let Err(io_error) = self.directory.save(key_file) { warn!("Error saving key file: {:?}", io_error); } @@ -210,7 +211,10 @@ mod tests { let temp = RandomTempPath::create_dir(); let mut sstore = SecretStore::new_test(&temp); - sstore.insert(H128::random(), "Cat".to_owned(), "pass"); + let id = H128::random(); + sstore.insert(id.clone(), "Cat".to_owned(), "pass"); + + assert!(sstore.get::(&id, "pass").is_ok()); } #[test] @@ -236,6 +240,25 @@ mod tests { } } + fn pregenerate_keys(temp: &RandomTempPath, count: usize) -> Vec { + use keys::directory::{KeyFileContent, KeyFileCrypto}; + let mut write_sstore = SecretStore::new_test(&temp); + let mut result = Vec::new(); + for _ in 0..count { + result.push(write_sstore.directory.save( + KeyFileContent::new( + KeyFileCrypto::new_pbkdf2( + FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(), + H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(), + H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(), + H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(), + 262144, + 32))) + .unwrap()); + } + result + } + #[test] fn secret_store_get() { let temp = RandomTempPath::create_dir(); @@ -259,5 +282,16 @@ mod tests { } } + #[test] + fn secret_store_delete() { + let temp = RandomTempPath::create_dir(); + let keys = pregenerate_keys(&temp, 5); + + let mut sstore = SecretStore::new_test(&temp); + sstore.delete(&keys[2]); + + assert_eq!(4, sstore.directory.list().unwrap().len()) + } + } From ca99679d1d009e8aad83cd4e0335cfef635724dc Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 11:57:13 +0300 Subject: [PATCH 09/19] fixed test names --- idea | 105 +++++++++++++++++++++++++++++++++++++++++ util/src/keys/store.rs | 8 ++-- 2 files changed, 109 insertions(+), 4 deletions(-) create mode 100755 idea diff --git a/idea b/idea new file mode 100755 index 000000000..c3e170967 --- /dev/null +++ b/idea @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import socket +import struct +import sys +import os +import time + +# see com.intellij.idea.SocketLock for the server side of this interface + +RUN_PATH = u'/home/nikvolf/idea/bin/idea.sh' +CONFIG_PATH = u'/home/nikvolf/.IdeaIC15/config' + +args = [] +skip_next = False +for i, arg in enumerate(sys.argv[1:]): + if arg == '-h' or arg == '-?' or arg == '--help': + print(('Usage:\n' + + ' {0} -h |-? | --help\n' + + ' {0} [-l|--line line] file[:line]\n' + + ' {0} diff ' + + ' {0} merge [base] ').format(sys.argv[0])) + exit(0) + elif arg == 'diff' and i == 0: + args.append(arg) + elif arg == 'merge' and i == 0: + args.append(arg) + elif arg == '-l' or arg == '--line': + args.append(arg) + skip_next = True + elif skip_next: + args.append(arg) + skip_next = False + else: + if ':' in arg: + file_path, line_number = arg.rsplit(':', 1) + if line_number.isdigit(): + args.append('-l') + args.append(line_number) + args.append(os.path.abspath(file_path)) + else: + args.append(os.path.abspath(arg)) + else: + args.append(os.path.abspath(arg)) + + +def launch_with_port(port): + found = False + + s = socket.socket() + s.settimeout(0.3) + try: + s.connect(('127.0.0.1', port)) + except: + return False + + while True: + try: + path_len = struct.unpack(">h", s.recv(2))[0] + path = s.recv(path_len) + if os.path.abspath(path) == os.path.abspath(CONFIG_PATH): + found = True + break + except: + break + + if found: + if args: + cmd = "activate " + os.getcwd() + "\0" + "\0".join(args) + encoded = struct.pack(">h", len(cmd)) + cmd + s.send(encoded) + time.sleep(0.5) # don't close socket immediately + return True + + return False + + +port = -1 +try: + f = open(os.path.join(CONFIG_PATH, 'port')) + port = int(f.read()) +except Exception: + type, value, traceback = sys.exc_info() + print('No IDE instance has been found. New one will be started.') + port = -1 + +if port == -1: + # SocketLock actually allows up to 50 ports, but the checking takes too long + for port in range(6942, 6942 + 10): + if launch_with_port(port): + exit() +else: + if launch_with_port(port): + exit() + +if sys.platform == "darwin": + # OS X: RUN_PATH is *.app path + if len(args): + args.insert(0, "--args") + os.execvp("open", ["-a", RUN_PATH] + args) +else: + # unix common + bin_dir, bin_file = os.path.split(RUN_PATH) + os.execv(RUN_PATH, [bin_file] + args) diff --git a/util/src/keys/store.rs b/util/src/keys/store.rs index e122ce808..b8b0b0a47 100644 --- a/util/src/keys/store.rs +++ b/util/src/keys/store.rs @@ -207,7 +207,7 @@ mod tests { use common::*; #[test] - fn secret_store_insert() { + fn can_insert() { let temp = RandomTempPath::create_dir(); let mut sstore = SecretStore::new_test(&temp); @@ -218,7 +218,7 @@ mod tests { } #[test] - fn secret_store_get_fail() { + fn can_get_fail() { let temp = RandomTempPath::create_dir(); { use keys::directory::{KeyFileContent, KeyFileCrypto}; @@ -260,7 +260,7 @@ mod tests { } #[test] - fn secret_store_get() { + fn can_get() { let temp = RandomTempPath::create_dir(); let key_id = { use keys::directory::{KeyFileContent, KeyFileCrypto}; @@ -283,7 +283,7 @@ mod tests { } #[test] - fn secret_store_delete() { + fn can_delete() { let temp = RandomTempPath::create_dir(); let keys = pregenerate_keys(&temp, 5); From 40f5ea400729e6cbbe3e596f8007e6f2c7593b4a Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 11:59:14 +0300 Subject: [PATCH 10/19] removed idea trash --- idea | 105 ----------------------------------------------------------- 1 file changed, 105 deletions(-) delete mode 100755 idea diff --git a/idea b/idea deleted file mode 100755 index c3e170967..000000000 --- a/idea +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import socket -import struct -import sys -import os -import time - -# see com.intellij.idea.SocketLock for the server side of this interface - -RUN_PATH = u'/home/nikvolf/idea/bin/idea.sh' -CONFIG_PATH = u'/home/nikvolf/.IdeaIC15/config' - -args = [] -skip_next = False -for i, arg in enumerate(sys.argv[1:]): - if arg == '-h' or arg == '-?' or arg == '--help': - print(('Usage:\n' + - ' {0} -h |-? | --help\n' + - ' {0} [-l|--line line] file[:line]\n' + - ' {0} diff ' + - ' {0} merge [base] ').format(sys.argv[0])) - exit(0) - elif arg == 'diff' and i == 0: - args.append(arg) - elif arg == 'merge' and i == 0: - args.append(arg) - elif arg == '-l' or arg == '--line': - args.append(arg) - skip_next = True - elif skip_next: - args.append(arg) - skip_next = False - else: - if ':' in arg: - file_path, line_number = arg.rsplit(':', 1) - if line_number.isdigit(): - args.append('-l') - args.append(line_number) - args.append(os.path.abspath(file_path)) - else: - args.append(os.path.abspath(arg)) - else: - args.append(os.path.abspath(arg)) - - -def launch_with_port(port): - found = False - - s = socket.socket() - s.settimeout(0.3) - try: - s.connect(('127.0.0.1', port)) - except: - return False - - while True: - try: - path_len = struct.unpack(">h", s.recv(2))[0] - path = s.recv(path_len) - if os.path.abspath(path) == os.path.abspath(CONFIG_PATH): - found = True - break - except: - break - - if found: - if args: - cmd = "activate " + os.getcwd() + "\0" + "\0".join(args) - encoded = struct.pack(">h", len(cmd)) + cmd - s.send(encoded) - time.sleep(0.5) # don't close socket immediately - return True - - return False - - -port = -1 -try: - f = open(os.path.join(CONFIG_PATH, 'port')) - port = int(f.read()) -except Exception: - type, value, traceback = sys.exc_info() - print('No IDE instance has been found. New one will be started.') - port = -1 - -if port == -1: - # SocketLock actually allows up to 50 ports, but the checking takes too long - for port in range(6942, 6942 + 10): - if launch_with_port(port): - exit() -else: - if launch_with_port(port): - exit() - -if sys.platform == "darwin": - # OS X: RUN_PATH is *.app path - if len(args): - args.insert(0, "--args") - os.execvp("open", ["-a", RUN_PATH] + args) -else: - # unix common - bin_dir, bin_file = os.path.split(RUN_PATH) - os.execv(RUN_PATH, [bin_file] + args) From aad49cb19f7205ad0f5c97c9e418df50a839b126 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 23:25:25 +0300 Subject: [PATCH 11/19] more functional --- util/src/keys/directory.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 7cc101da1..f6d9c8e22 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -510,19 +510,16 @@ impl KeyDirectory { cache_usage.drain(..untracked_usages); } - let mut cache = self.cache.borrow_mut(); - if cache.len() <= MAX_CACHE_USAGE_TRACK { return; } + if self.cache.borrow().len() <= MAX_CACHE_USAGE_TRACK { return; } let uniqs: HashSet<&Uuid> = cache_usage.iter().collect(); - let mut removes = HashSet::new(); - - for key in cache.keys() { - if !uniqs.contains(key) { - removes.insert(key.clone()); - } + let removes: Vec = { + let cache = self.cache.borrow(); + cache.keys().cloned().filter(|key| !uniqs.contains(key)).collect() + }; + for key in removes { + self.cache.borrow_mut().remove(&key); } - - for removed_key in removes { cache.remove(&removed_key); } } /// Reports how many keys are currently cached. From 95c74dbd30e092c24f142c4e17ada412de7d4dec Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 23:34:55 +0300 Subject: [PATCH 12/19] more func --- util/src/keys/directory.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index f6d9c8e22..8ebf2c27d 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -513,13 +513,13 @@ impl KeyDirectory { if self.cache.borrow().len() <= MAX_CACHE_USAGE_TRACK { return; } let uniqs: HashSet<&Uuid> = cache_usage.iter().collect(); - let removes: Vec = { + let removes:Vec = { let cache = self.cache.borrow(); cache.keys().cloned().filter(|key| !uniqs.contains(key)).collect() }; - for key in removes { - self.cache.borrow_mut().remove(&key); - } + if removes.is_empty() { return } + let mut cache = self.cache.borrow_mut(); + for key in removes { cache.remove(&key); } } /// Reports how many keys are currently cached. From fffd93607d488575737710a810aa06b5d879d53f Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Wed, 17 Feb 2016 23:38:16 +0300 Subject: [PATCH 13/19] just in case check --- util/src/keys/directory.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 8ebf2c27d..9c01d1974 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -517,7 +517,7 @@ impl KeyDirectory { let cache = self.cache.borrow(); cache.keys().cloned().filter(|key| !uniqs.contains(key)).collect() }; - if removes.is_empty() { return } + if removes.is_empty() { return; } let mut cache = self.cache.borrow_mut(); for key in removes { cache.remove(&key); } } @@ -1093,6 +1093,14 @@ mod directory_tests { // since all keys are different, should be exactly MAX_CACHE_USAGE_TRACK assert_eq!(MAX_CACHE_USAGE_TRACK, directory.cache_size()) } + + #[test] + fn collects_garbage_on_empty() { + let temp_path = RandomTempPath::create_dir(); + let mut directory = KeyDirectory::new(&temp_path.as_path()); + directory.collect_garbage(); + assert_eq!(0, directory.cache_size()) + } } #[cfg(test)] From a4846e4aad65be53c0f700af269ca05a74c9626f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 19 Feb 2016 11:31:40 +0100 Subject: [PATCH 14/19] Update bytes.rs --- util/src/bytes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/src/bytes.rs b/util/src/bytes.rs index a26804c5b..08c299ddf 100644 --- a/util/src/bytes.rs +++ b/util/src/bytes.rs @@ -141,7 +141,7 @@ impl<'a> Deref for BytesRef<'a> { fn deref(&self) -> &[u8] { match *self { BytesRef::Flexible(ref bytes) => bytes, - BytesRef::Fixed(ref bytes) => bytes + BytesRef::Fixed(ref bytes) => bytes, } } } @@ -150,7 +150,7 @@ impl <'a> DerefMut for BytesRef<'a> { fn deref_mut(&mut self) -> &mut [u8] { match *self { BytesRef::Flexible(ref mut bytes) => bytes, - BytesRef::Fixed(ref mut bytes) => bytes + BytesRef::Fixed(ref mut bytes) => bytes, } } } From f50bf528e6106960f8b8620eb59df195d1334a82 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 19 Feb 2016 11:57:52 +0100 Subject: [PATCH 15/19] fixed allow warnings in util --- util/src/lib.rs | 9 +++++---- util/src/network/discovery.rs | 2 +- util/src/network/host.rs | 6 +++--- util/src/panics.rs | 2 +- util/src/trie/triedb.rs | 2 +- util/src/trie/triedbmut.rs | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/util/src/lib.rs b/util/src/lib.rs index ef76bb885..59713a107 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -17,15 +17,16 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] #![cfg_attr(feature="dev", plugin(clippy))] + // Clippy settings // TODO [todr] not really sure -#![allow(needless_range_loop)] +#![cfg_attr(feature="dev", allow(needless_range_loop))] // Shorter than if-else -#![allow(match_bool)] +#![cfg_attr(feature="dev", allow(match_bool))] // We use that to be more explicit about handled cases -#![allow(match_same_arms)] +#![cfg_attr(feature="dev", allow(match_same_arms))] // Keeps consistency (all lines with `.clone()`) and helpful when changing ref to non-ref. -#![allow(clone_on_copy)] +#![cfg_attr(feature="dev", allow(clone_on_copy))] //! Ethcore-util library //! diff --git a/util/src/network/discovery.rs b/util/src/network/discovery.rs index 3e914761d..e28c79e80 100644 --- a/util/src/network/discovery.rs +++ b/util/src/network/discovery.rs @@ -138,7 +138,7 @@ impl Discovery { ret } - #[allow(cyclomatic_complexity)] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b [NodeBucket]) -> Vec<&'b NodeId> { // send ALPHA FindNode packets to nodes we know, closest to target diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 430850453..5db724d32 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -471,7 +471,7 @@ impl Host where Message: Send + Sync + Clone { } } - #[allow(single_match)] + #[cfg_attr(feature="dev", allow(single_match))] fn connect_peer(&self, id: &NodeId, io: &IoContext>) { if self.have_session(id) { @@ -501,7 +501,7 @@ impl Host where Message: Send + Sync + Clone { self.create_connection(socket, Some(id), io); } - #[allow(block_in_if_condition_stmt)] + #[cfg_attr(feature="dev", allow(block_in_if_condition_stmt))] fn create_connection(&self, socket: TcpStream, id: Option<&NodeId>, io: &IoContext>) { let nonce = self.info.write().unwrap().next_nonce(); let mut connections = self.connections.write().unwrap(); @@ -532,7 +532,7 @@ impl Host where Message: Send + Sync + Clone { io.update_registration(TCP_ACCEPT).expect("Error registering TCP listener"); } - #[allow(single_match)] + #[cfg_attr(feature="dev", allow(single_match))] fn connection_writable(&self, token: StreamToken, io: &IoContext>) { let mut create_session = false; let mut kill = false; diff --git a/util/src/panics.rs b/util/src/panics.rs index 6bd8fc1d6..18ed9ecb1 100644 --- a/util/src/panics.rs +++ b/util/src/panics.rs @@ -72,7 +72,7 @@ impl PanicHandler { /// Invoke closure and catch any possible panics. /// In case of panic notifies all listeners about it. - #[allow(deprecated)] + #[cfg_attr(feature="dev", allow(deprecated))] // TODO [todr] catch_panic is deprecated but panic::recover has different bounds (not allowing mutex) pub fn catch_panic(&self, g: G) -> thread::Result where G: FnOnce() -> R + Send + 'static { let guard = PanicGuard { handler: self }; diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index e7884a177..c4b5e120c 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -54,7 +54,7 @@ pub struct TrieDB<'db> { pub hash_count: usize, } -#[allow(wrong_self_convention)] +#[cfg_attr(feature="dev", allow(wrong_self_convention))] impl<'db> TrieDB<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 8e92063aa..2b4567264 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -66,7 +66,7 @@ enum MaybeChanged<'a> { Changed(Bytes), } -#[allow(wrong_self_convention)] +#[cfg_attr(feature="dev", allow(wrong_self_convention))] impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -350,7 +350,7 @@ impl<'db> TrieDBMut<'db> { } } - #[allow(cyclomatic_complexity)] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that includes the new node. From b5d6359030d5b832d913776796e3fe56ca719699 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 19 Feb 2016 12:19:43 +0100 Subject: [PATCH 16/19] fixed allow warnings in ethcore --- ethcore/src/basic_types.rs | 2 +- ethcore/src/block.rs | 2 +- ethcore/src/block_queue.rs | 2 +- ethcore/src/ethereum/ethash.rs | 2 +- ethcore/src/evm/interpreter.rs | 6 +++--- ethcore/src/externalities.rs | 2 +- ethcore/src/lib.rs | 8 +++++--- ethcore/src/service.rs | 4 ++-- ethcore/src/spec.rs | 2 +- ethcore/src/state.rs | 2 +- rpc/src/v1/types/block_number.rs | 5 ++--- sync/src/chain.rs | 4 ++-- sync/src/lib.rs | 3 ++- 13 files changed, 23 insertions(+), 21 deletions(-) diff --git a/ethcore/src/basic_types.rs b/ethcore/src/basic_types.rs index 2e9c5d7b9..5f6515c0d 100644 --- a/ethcore/src/basic_types.rs +++ b/ethcore/src/basic_types.rs @@ -24,7 +24,7 @@ pub type LogBloom = H2048; /// Constant 2048-bit datum for 0. Often used as a default. pub static ZERO_LOGBLOOM: LogBloom = H2048([0x00; 256]); -#[allow(enum_variant_names)] +#[cfg_attr(feature="dev", allow(enum_variant_names))] /// Semantic boolean for when a seal/signature is included. pub enum Seal { /// The seal/signature is included. diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index d72fbd1ae..b7590143e 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -16,7 +16,7 @@ //! Blockchain block. -#![allow(ptr_arg)] // Because of &LastHashes -> &Vec<_> +#![cfg_attr(feature="dev", allow(ptr_arg))] // Because of &LastHashes -> &Vec<_> use common::*; use engine::*; diff --git a/ethcore/src/block_queue.rs b/ethcore/src/block_queue.rs index 1a1dee48e..851c56c05 100644 --- a/ethcore/src/block_queue.rs +++ b/ethcore/src/block_queue.rs @@ -87,7 +87,7 @@ struct QueueSignal { } impl QueueSignal { - #[allow(bool_comparison)] + #[cfg_attr(feature="dev", allow(bool_comparison))] fn set(&self) { if self.signalled.compare_and_swap(false, true, AtomicOrdering::Relaxed) == false { self.message_channel.send(UserMessage(SyncMessage::BlockVerified)).expect("Error sending BlockVerified message"); diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index c45b22102..ff6c6ba72 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -206,7 +206,7 @@ impl Engine for Ethash { } } -#[allow(wrong_self_convention)] // to_ethash should take self +#[cfg_attr(feature="dev", allow(wrong_self_convention))] // to_ethash should take self impl Ethash { fn calculate_difficuty(&self, header: &Header, parent: &Header) -> U256 { const EXP_DIFF_PERIOD: u64 = 100000; diff --git a/ethcore/src/evm/interpreter.rs b/ethcore/src/evm/interpreter.rs index 7efd79d00..e09c37319 100644 --- a/ethcore/src/evm/interpreter.rs +++ b/ethcore/src/evm/interpreter.rs @@ -243,7 +243,7 @@ struct CodeReader<'a> { code: &'a Bytes } -#[allow(len_without_is_empty)] +#[cfg_attr(feature="dev", allow(len_without_is_empty))] impl<'a> CodeReader<'a> { /// Get `no_of_bytes` from code and convert to U256. Move PC fn read(&mut self, no_of_bytes: usize) -> U256 { @@ -258,7 +258,7 @@ impl<'a> CodeReader<'a> { } } -#[allow(enum_variant_names)] +#[cfg_attr(feature="dev", allow(enum_variant_names))] enum InstructionCost { Gas(U256), GasMem(U256, U256), @@ -347,7 +347,7 @@ impl evm::Evm for Interpreter { } impl Interpreter { - #[allow(cyclomatic_complexity)] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] fn get_gas_cost_mem(&self, ext: &evm::Ext, instruction: Instruction, diff --git a/ethcore/src/externalities.rs b/ethcore/src/externalities.rs index 360bd9738..9116629fb 100644 --- a/ethcore/src/externalities.rs +++ b/ethcore/src/externalities.rs @@ -188,7 +188,7 @@ impl<'a> Ext for Externalities<'a> { self.state.code(address).unwrap_or_else(|| vec![]) } - #[allow(match_ref_pats)] + #[cfg_attr(feature="dev", allow(match_ref_pats))] fn ret(&mut self, gas: &U256, data: &[u8]) -> Result { match &mut self.output { &mut OutputPolicy::Return(BytesRef::Fixed(ref mut slice)) => unsafe { diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 11d129f61..8e59cf7a2 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -17,12 +17,14 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] #![cfg_attr(feature="dev", plugin(clippy))] + +// Clippy config // TODO [todr] not really sure -#![allow(needless_range_loop)] +#![cfg_attr(feature="dev", allow(needless_range_loop))] // Shorter than if-else -#![allow(match_bool)] +#![cfg_attr(feautre="dev", allow(match_bool))] // Keeps consistency (all lines with `.clone()`) and helpful when changing ref to non-ref. -#![allow(clone_on_copy)] +#![cfg_attr(feature="dev", allow(clone_on_copy))] //! Ethcore library //! diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 5e68efae9..d20959077 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -110,8 +110,8 @@ impl IoHandler for ClientIoHandler { } } - #[allow(match_ref_pats)] - #[allow(single_match)] + #[cfg_attr(feature="dev", allow(match_ref_pats))] + #[cfg_attr(feature="dev", allow(single_match))] fn message(&self, io: &IoContext, net_message: &NetSyncMessage) { if let &UserMessage(ref message) = net_message { match message { diff --git a/ethcore/src/spec.rs b/ethcore/src/spec.rs index 67269d334..5714ca734 100644 --- a/ethcore/src/spec.rs +++ b/ethcore/src/spec.rs @@ -97,7 +97,7 @@ pub struct Spec { genesis_state: PodState, } -#[allow(wrong_self_convention)] // because to_engine(self) should be to_engine(&self) +#[cfg_attr(feature="dev", allow(wrong_self_convention))] // because to_engine(self) should be to_engine(&self) impl Spec { /// Convert this object into a boxed Engine of the right underlying type. // TODO avoid this hard-coded nastiness - use dynamic-linked plugin framework instead. diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 3eaedb9bf..bbec8cd37 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -224,7 +224,7 @@ impl State { /// Commit accounts to SecTrieDBMut. This is similar to cpp-ethereum's dev::eth::commit. /// `accounts` is mutable because we may need to commit the code or storage and record that. - #[allow(match_ref_pats)] + #[cfg_attr(feature="dev", allow(match_ref_pats))] pub fn commit_into(db: &mut HashDB, root: &mut H256, accounts: &mut HashMap>) { // first, commit the sub trees. // TODO: is this necessary or can we dispense with the `ref mut a` for just `a`? diff --git a/rpc/src/v1/types/block_number.rs b/rpc/src/v1/types/block_number.rs index 546816eba..bb563de99 100644 --- a/rpc/src/v1/types/block_number.rs +++ b/rpc/src/v1/types/block_number.rs @@ -55,13 +55,12 @@ impl Visitor for BlockNumberVisitor { } impl Into for BlockNumber { - #[allow(match_same_arms)] fn into(self) -> BlockId { match self { BlockNumber::Num(n) => BlockId::Number(n), BlockNumber::Earliest => BlockId::Earliest, - BlockNumber::Latest => BlockId::Latest, - BlockNumber::Pending => BlockId::Latest // TODO: change this once blockid support pending + // TODO: change this once blockid support pendingst, + BlockNumber::Pending | BlockNumber::Latest => BlockId::Latest, } } } diff --git a/sync/src/chain.rs b/sync/src/chain.rs index 1c830ba7e..00e677538 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -251,7 +251,7 @@ impl ChainSync { } - #[allow(for_kv_map)] // Because it's not possible to get `values_mut()` + #[cfg_attr(feature="dev", allow(for_kv_map))] // Because it's not possible to get `values_mut()` /// Rest sync. Clear all downloaded data but keep the queue fn reset(&mut self) { self.downloading_headers.clear(); @@ -319,7 +319,7 @@ impl ChainSync { Ok(()) } - #[allow(cyclomatic_complexity)] + #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] /// Called by peer once it has new block headers during sync fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &UntrustedRlp) -> Result<(), PacketDecodeError> { self.reset_peer_asking(peer_id, PeerAsking::BlockHeaders); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index d6bce2447..fd586409a 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -17,8 +17,9 @@ #![warn(missing_docs)] #![cfg_attr(feature="dev", feature(plugin))] #![cfg_attr(feature="dev", plugin(clippy))] + // Keeps consistency (all lines with `.clone()`) and helpful when changing ref to non-ref. -#![allow(clone_on_copy)] +#![cfg_attr(feature="dev", allow(clone_on_copy))] //! Blockchain sync module //! Implements ethereum protocol version 63 as specified here: From 63c5a2e58f6a03835ad12690b58b3874be31f1a0 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 19 Feb 2016 14:52:29 +0300 Subject: [PATCH 17/19] copy instead of ref-map --- util/src/keys/directory.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 9c01d1974..67bf8cbc1 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -473,7 +473,7 @@ impl KeyDirectory { /// Returns key given by id if corresponding file exists and no load error occured. /// Warns if any error occured during the key loading - pub fn get(&self, id: &Uuid) -> Option> { + pub fn get(&self, id: &Uuid) -> Option { let path = self.key_path(id); { let mut usage = self.cache_usage.borrow_mut(); @@ -492,7 +492,10 @@ impl KeyDirectory { } } - Some(Ref::map(self.cache.borrow(), |c| c.get(id).expect("should be they key, we have just inserted or checked it"))) + // todo: replace with Ref::map when it stabilized to avoid copies + Some(self.cache.borrow().get(id) + .expect("should be they key, we have just inserted or checked it") + .clone()) } /// Returns current path to the directory with keys From fe979f0d46c5d5be41ff3dc71f1a83bc9e71c249 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 19 Feb 2016 15:21:52 +0300 Subject: [PATCH 18/19] [ci skip] update readme with rust override --- README.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b840195d8..29f634c16 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,17 @@ apt-get install -y --force-yes librocksdb-dev # install multirust curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sh -s -- --yes -# install beta and make it default -multirust default beta +# install beta +multirust update beta # download and build parity git clone https://github.com/ethcore/parity cd parity + +# parity should be build with rust beta +multirust override beta + +# build in release cargo build --release ``` @@ -49,12 +54,17 @@ cd .. # install rust beta curl -sf https://raw.githubusercontent.com/brson/multirust/master/blastoff.sh | sudo sh -s -- --yes -# install beta and make it default -sudo multirust default beta +# install rust beta +sudo multirust update beta # download and build parity git clone https://github.com/ethcore/parity cd parity + +# parity should be build with rust beta +sudo multirust override beta + +# build in release cargo build --release ``` @@ -66,12 +76,16 @@ brew update brew install rocksdb brew install multirust -# install beta and make it default -multirust default beta +# install beta +multirust update beta # download and build parity git clone https://github.com/ethcore/parity cd parity + +# use rust beta for building parity +multirust override beta + cargo build --release ``` From d43b23d663ccb401352efbd8854745b1ecadf079 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 19 Feb 2016 13:30:36 +0100 Subject: [PATCH 19/19] Update directory.rs --- util/src/keys/directory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/keys/directory.rs b/util/src/keys/directory.rs index 67bf8cbc1..23c483482 100644 --- a/util/src/keys/directory.rs +++ b/util/src/keys/directory.rs @@ -494,7 +494,7 @@ impl KeyDirectory { // todo: replace with Ref::map when it stabilized to avoid copies Some(self.cache.borrow().get(id) - .expect("should be they key, we have just inserted or checked it") + .expect("Key should be there, we have just inserted or checked it.") .clone()) }