Merge pull request #509 from ethcore/geth-import-no-utility-crate

Account management + geth keystore import (no utility crate added)

util/res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9

@@ -0,0 +1,21 @@
+{
+	"address": "3f49624084b67849c7b4e805c5988c21a430f9d9",
+	"Crypto": {
+		"cipher": "aes-128-ctr",
+		"ciphertext": "9f27e3dd4fc73e7103ed61e5493662189a3eb52223ae49e3d1deacc04c889eae",
+		"cipherparams": {
+			"iv": "457494bf05f2618c397dc74dbb5181c0"
+		},
+		"kdf": "scrypt",
+		"kdfparams": {
+			"dklen": 32,
+			"n": 262144,
+			"p": 1,
+			"r": 8,
+			"salt": "db14edb18c41ee7f5ec4397df89c3a2ae4d0af60884c52bb54ce490574f8df33"
+		},
+		"mac": "572d24532438d31fdf513c744a3ff26c933ffda5744ee42bc71661cbe3f2112e"
+	},
+	"id": "62a0ad73-556d-496a-8e1c-0783d30d3ace",
+	"version": 3
+}

util/res/geth_keystore/UTC--2016-02-20T09-33-03.984382741Z--5ba4dcf897e97c2bdf8315b9ef26c13c085988cf

@@ -0,0 +1,21 @@
+{
+	"address": "5ba4dcf897e97c2bdf8315b9ef26c13c085988cf",
+	"Crypto": {
+		"cipher": "aes-128-ctr",
+		"ciphertext": "d4a08ec930163778273920f6ad1d49b71836337be6fd9863993ac700a612fddd",
+		"cipherparams": {
+			"iv": "89ce5ec129fc27cd5bcbeb8c92bdad50"
+		},
+		"kdf": "scrypt",
+		"kdfparams": {
+			"dklen": 32,
+			"n": 262144,
+			"p": 1,
+			"r": 8,
+			"salt": "612ab108dc37e69ee8af37a7b24bf7f2234086d7bbf945bacdeccce331f7f84a"
+		},
+		"mac": "4152caa7444e06784223d735cea80cd2690b4c587ad8db3d5529442227b25695"
+	},
+	"id": "35086353-fb12-4029-b56b-033cd61ce35b",
+	"version": 3
+}

util/src/hash.rs

@@ -235,7 +235,7 @@ macro_rules! impl_hash {
 		}
 
 		impl serde::Serialize for $from {
-			fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error> 
+			fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
 			where S: serde::Serializer {
 				let mut hex = "0x".to_owned();
 				hex.push_str(self.to_hex().as_ref());
@@ -250,7 +250,7 @@ macro_rules! impl_hash {
 
 				impl serde::de::Visitor for HashVisitor {
 					type Value = $from;
-					
+
 					fn visit_str<E>(&mut self, value: &str) -> Result<Self::Value, E> where E: serde::Error {
 						// 0x + len
 						if value.len() != 2 + $size * 2 {
@@ -719,4 +719,3 @@ mod tests {
 		assert_eq!(r, u);
 	}
 }
-

util/src/keys/directory.rs

@@ -333,7 +333,9 @@ pub struct KeyFileContent {
 	/// Holds cypher and decrypt function settings.
 	pub crypto: KeyFileCrypto,
 	/// The identifier.
-	pub id: Uuid
+	pub id: Uuid,
+	/// Account (if present)
+	pub account: Option<Address>,
 }
 
 #[derive(Debug)]
@@ -374,7 +376,19 @@ impl KeyFileContent {
 		KeyFileContent {
 			id: new_uuid(),
 			version: KeyFileVersion::V3(3),
-			crypto: crypto
+			crypto: crypto,
+			account: None
+		}
+	}
+
+	/// Loads key from valid json, returns error and records warning if key is mallformed
+	pub fn load(json: &Json) -> Result<KeyFileContent, ()> {
+		match Self::from_json(json) {
+			Ok(key_file) => Ok(key_file),
+			Err(e) => {
+				warn!(target: "sstore", "Error parsing json for key: {:?}", e);
+				Err(())
+			}
 		}
 	}
 
@@ -407,6 +421,9 @@ impl KeyFileContent {
 			Ok(id) => id
 		};
 
+		let account = as_object.get("address").and_then(|json| json.as_string()).and_then(
+			|account_text| match Address::from_str(account_text) { Ok(account) => Some(account), Err(_) => None });
+
 		let crypto = match as_object.get("crypto") {
 			None => { return Err(KeyFileParseError::NoCryptoSection); }
 			Some(crypto_json) => match KeyFileCrypto::from_json(crypto_json) {
@@ -418,7 +435,8 @@ impl KeyFileContent {
 		Ok(KeyFileContent {
 			version: version,
 			id: id.clone(),
-			crypto: crypto
+			crypto: crypto,
+			account: account
 		})
 	}
 
@@ -427,6 +445,7 @@ impl KeyFileContent {
 		map.insert("id".to_owned(), Json::String(uuid_to_string(&self.id)));
 		map.insert("version".to_owned(), Json::U64(CURRENT_DECLARED_VERSION));
 		map.insert("crypto".to_owned(), self.crypto.to_json());
+		if let Some(ref address) = self.account { map.insert("address".to_owned(), Json::String(format!("{:?}", address))); }
 		Json::Object(map)
 	}
 }
@@ -599,6 +618,8 @@ impl KeyDirectory {
 			Err(_) => Err(KeyFileLoadError::ParseError(KeyFileParseError::InvalidJson))
 		}
 	}
+
+
 }
 
 
@@ -653,7 +674,7 @@ mod file_tests {
 	}
 
 	#[test]
-	fn can_read_scrypt_krf() {
+	fn can_read_scrypt_kdf() {
 		let json = Json::from_str(
 			r#"
 				{
@@ -689,6 +710,47 @@ mod file_tests {
 		}
 	}
 
+	#[test]
+	fn can_read_scrypt_kdf_params() {
+		let json = Json::from_str(
+			r#"
+				{
+					"crypto" : {
+						"cipher" : "aes-128-ctr",
+						"cipherparams" : {
+							"iv" : "83dbcc02d8ccb40e466191a123791e0e"
+						},
+						"ciphertext" : "d172bf743a674da9cdad04534d56926ef8358534d458fffccd4e6ad2fbde479c",
+						"kdf" : "scrypt",
+						"kdfparams" : {
+							"dklen" : 32,
+							"n" : 262144,
+							"r" : 1,
+							"p" : 8,
+							"salt" : "ab0c7876052600dd703518d6fc3fe8984592145b591fc8fb5c6d43190334ba19"
+						},
+						"mac" : "2103ac29920d71da29f15d75b4a16dbe95cfd7ff8faea1056c33131d846e3097"
+					},
+					"id" : "3198bc9c-6672-5ab3-d995-4942343ae5b6",
+					"version" : 3
+				}
+			"#).unwrap();
+
+		match KeyFileContent::from_json(&json) {
+			Ok(key_file) => {
+				match key_file.crypto.kdf {
+					KeyFileKdf::Scrypt(scrypt_params) => {
+						assert_eq!(262144, scrypt_params.n);
+						assert_eq!(1, scrypt_params.r);
+						assert_eq!(8, scrypt_params.p);
+					},
+					_ => { panic!("expected kdf params of crypto to be of scrypt type" ); }
+				}
+			},
+			Err(e) => panic!("Error parsing valid file: {:?}", e)
+		}
+	}
+
 	#[test]
 	fn can_return_error_no_id() {
 		let json = Json::from_str(
@@ -844,7 +906,7 @@ mod file_tests {
 				panic!("Should be error of no identifier, got ok");
 			},
 			Err(KeyFileParseError::Crypto(CryptoParseError::Scrypt(_))) => { },
-			Err(other_error) => { panic!("should be error of no identifier, got {:?}", other_error); }
+			Err(other_error) => { panic!("should be scrypt parse error, got {:?}", other_error); }
 		}
 	}
 

util/src/keys/geth_import.rs

@@ -0,0 +1,165 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Geth keys import/export tool
+
+use common::*;
+use keys::store::SecretStore;
+use keys::directory::KeyFileContent;
+
+/// Enumerates all geth keys in the directory and returns collection of tuples `(accountId, filename)`
+pub fn enumerate_geth_keys(path: &Path) -> Result<Vec<(Address, String)>, io::Error> {
+	let mut entries = Vec::new();
+	for entry in try!(fs::read_dir(path)) {
+		let entry = try!(entry);
+		if !try!(fs::metadata(entry.path())).is_dir() {
+			match entry.file_name().to_str() {
+				Some(name) => {
+					let parts: Vec<&str> = name.split("--").collect();
+					if parts.len() != 3 { continue; }
+					match Address::from_str(parts[2]) {
+						Ok(account_id) => { entries.push((account_id, name.to_owned())); }
+						Err(e) => { panic!("error: {:?}", e); }
+					}
+				},
+				None => { continue; }
+			};
+		}
+	}
+	Ok(entries)
+}
+
+/// Geth import error
+#[derive(Debug)]
+pub enum ImportError {
+	/// Io error reading geth file
+	IoError(io::Error),
+	/// format error
+	FormatError,
+}
+
+impl From<io::Error> for ImportError {
+	fn from (err: io::Error) -> ImportError {
+		ImportError::IoError(err)
+	}
+}
+
+/// Imports one geth key to the store
+pub fn import_geth_key(secret_store: &mut SecretStore, geth_keyfile_path: &Path) -> Result<(), ImportError> {
+	let mut file = try!(fs::File::open(geth_keyfile_path));
+	let mut buf = String::new();
+	try!(file.read_to_string(&mut buf));
+
+	let mut json_result = Json::from_str(&buf);
+	let mut json = match json_result {
+		Ok(ref mut parsed_json) => try!(parsed_json.as_object_mut().ok_or(ImportError::FormatError)),
+		Err(_) => { return Err(ImportError::FormatError); }
+	};
+	let crypto_object = try!(json.get("Crypto").and_then(|crypto| crypto.as_object()).ok_or(ImportError::FormatError)).clone();
+	json.insert("crypto".to_owned(), Json::Object(crypto_object));
+	json.remove("Crypto");
+	match KeyFileContent::load(&Json::Object(json.clone())) {
+		Ok(key_file) => try!(secret_store.import_key(key_file)),
+		Err(_) => { return Err(ImportError::FormatError); }
+	};
+	Ok(())
+}
+
+/// Imports all geth keys in the directory
+pub fn import_geth_keys(secret_store: &mut SecretStore, geth_keyfiles_directory: &Path) -> Result<(), ImportError> {
+	use std::path::PathBuf;
+	let geth_files = try!(enumerate_geth_keys(geth_keyfiles_directory));
+	for &(ref address, ref file_path) in geth_files.iter() {
+		let mut path = PathBuf::new();
+		path.push(geth_keyfiles_directory);
+		path.push(file_path);
+		if let Err(e) = import_geth_key(secret_store, Path::new(&path)) {
+			warn!("Skipped geth address {}, error importing: {:?}", address, e)
+		}
+	}
+	Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use common::*;
+	use keys::store::SecretStore;
+
+
+	#[test]
+	fn can_enumerate() {
+		let keys = enumerate_geth_keys(Path::new("res/geth_keystore")).unwrap();
+		assert_eq!(2, keys.len());
+	}
+
+	#[test]
+	fn can_import() {
+		let temp = ::devtools::RandomTempPath::create_dir();
+		let mut secret_store = SecretStore::new_in(temp.as_path());
+		import_geth_key(&mut secret_store, Path::new("res/geth_keystore/UTC--2016-02-17T09-20-45.721400158Z--3f49624084b67849c7b4e805c5988c21a430f9d9")).unwrap();
+		let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
+		assert!(key.is_some());
+	}
+
+	#[test]
+	fn can_import_directory() {
+		let temp = ::devtools::RandomTempPath::create_dir();
+		let mut secret_store = SecretStore::new_in(temp.as_path());
+		import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap();
+
+		let key = secret_store.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
+		assert!(key.is_some());
+
+		let key = secret_store.account(&Address::from_str("5ba4dcf897e97c2bdf8315b9ef26c13c085988cf").unwrap());
+		assert!(key.is_some());
+	}
+
+	#[test]
+	fn imports_as_scrypt_keys() {
+		use keys::directory::{KeyDirectory, KeyFileKdf};
+		let temp = ::devtools::RandomTempPath::create_dir();
+		{
+			let mut secret_store = SecretStore::new_in(temp.as_path());
+			import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap();
+		}
+
+		let key_directory = KeyDirectory::new(&temp.as_path());
+		let key_file = key_directory.get(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap()).unwrap();
+
+		match key_file.crypto.kdf {
+			KeyFileKdf::Scrypt(scrypt_params) => {
+				assert_eq!(262144, scrypt_params.n);
+				assert_eq!(8, scrypt_params.r);
+				assert_eq!(1, scrypt_params.p);
+			},
+			_ => { panic!("expected kdf params of crypto to be of scrypt type" ); }
+		}
+	}
+
+	#[test]
+	fn can_decrypt_with_imported() {
+		use keys::store::EncryptedHashMap;
+
+		let temp = ::devtools::RandomTempPath::create_dir();
+		let mut secret_store = SecretStore::new_in(temp.as_path());
+		import_geth_keys(&mut secret_store, Path::new("res/geth_keystore")).unwrap();
+
+		let val = secret_store.get::<Bytes>(&H128::from_str("62a0ad73556d496a8e1c0783d30d3ace").unwrap(), "123");
+		assert!(val.is_ok());
+		assert_eq!(32, val.unwrap().len());
+	}
+}

util/src/keys/mod.rs

@@ -18,3 +18,4 @@
 
 pub mod directory;
 pub mod store;
+mod geth_import;

util/src/keys/store.rs

@@ -19,11 +19,12 @@
 use keys::directory::*;
 use common::*;
 use rcrypto::pbkdf2::*;
+use rcrypto::scrypt::*;
 use rcrypto::hmac::*;
 use crypto;
 
 const KEY_LENGTH: u32 = 32;
-const KEY_ITERATIONS: u32 = 4096;
+const KEY_ITERATIONS: u32 = 10240;
 const KEY_LENGTH_AES: u32 = KEY_LENGTH/2;
 
 const KEY_LENGTH_USIZE: usize = KEY_LENGTH as usize;
@@ -60,15 +61,62 @@ pub struct SecretStore {
 }
 
 impl SecretStore {
-	/// new instance of Secret Store
+	/// new instance of Secret Store in default home directory
 	pub fn new() -> SecretStore {
 		let mut path = ::std::env::home_dir().expect("Failed to get home dir");
-		path.push(".keys");
+		path.push(".parity");
+		path.push("keys");
+		Self::new_in(&path)
+	}
+
+	/// new instance of Secret Store in specific directory
+	pub fn new_in(path: &Path) -> SecretStore {
 		SecretStore {
-			directory: KeyDirectory::new(&path)
+			directory: KeyDirectory::new(path)
 		}
 	}
 
+	/// trys to import keys in the known locations
+	pub fn try_import_existing(&mut self) {
+		use std::path::PathBuf;
+		use keys::geth_import;
+
+		let mut import_path = PathBuf::new();
+		import_path.push(::std::env::home_dir().expect("Failed to get home dir"));
+		import_path.push(".ethereum");
+		import_path.push("keystore");
+		if let Err(e) = geth_import::import_geth_keys(self, &import_path) {
+			warn!(target: "sstore", "Error retrieving geth keys: {:?}", e)
+		}
+	}
+
+	/// Lists all accounts and corresponding key ids
+	pub fn accounts(&self) -> Result<Vec<(Address, H128)>, ::std::io::Error> {
+		let accounts = try!(self.directory.list()).iter().map(|key_id| self.directory.get(key_id))
+			.filter(|key| key.is_some())
+			.map(|key| { let some_key = key.unwrap(); (some_key.account, some_key.id) })
+			.filter(|&(ref account, _)| account.is_some())
+			.map(|(account, id)| (account.unwrap(), id))
+			.collect::<Vec<(Address, H128)>>();
+		Ok(accounts)
+	}
+
+	/// Resolves key_id by account address
+	pub fn account(&self, account: &Address) -> Option<H128> {
+		let mut accounts = match self.accounts() {
+			Ok(accounts) => accounts,
+			Err(e) => { warn!(target: "sstore", "Failed to load accounts: {}", e); return None; }
+		};
+		accounts.retain(|&(ref store_account, _)| account == store_account);
+		accounts.first().and_then(|&(_, ref key_id)| Some(key_id.clone()))
+	}
+
+	/// Imports pregenerated key, returns error if not saved correctly
+	pub fn import_key(&mut self, key_file: KeyFileContent) -> Result<(), ::std::io::Error> {
+		try!(self.directory.save(key_file));
+		Ok(())
+	}
+
 	#[cfg(test)]
 	fn new_test(path: &::devtools::RandomTempPath) -> SecretStore {
 		SecretStore {
@@ -90,6 +138,15 @@ fn derive_key(password: &str, salt: &H256) -> (Bytes, Bytes) {
 	derive_key_iterations(password, salt, KEY_ITERATIONS)
 }
 
+fn derive_key_scrypt(password: &str, salt: &H256, n: u32, p: u32, r: u32) -> (Bytes, Bytes) {
+	let mut derived_key = vec![0u8; KEY_LENGTH_USIZE];
+	let scrypt_params = ScryptParams::new(n.trailing_zeros() as u8, r, p);
+	scrypt(password.as_bytes(), &salt.as_slice(), &scrypt_params, &mut derived_key);
+	let derived_right_bits = &derived_key[0..KEY_LENGTH_AES_USIZE];
+	let derived_left_bits = &derived_key[KEY_LENGTH_AES_USIZE..KEY_LENGTH_USIZE];
+	(derived_right_bits.to_vec(), derived_left_bits.to_vec())
+}
+
 fn derive_mac(derived_left_bits: &[u8], cipher_text: &[u8]) -> Bytes {
 	let mut mac = vec![0u8; KEY_LENGTH_AES_USIZE + cipher_text.len()];
 	mac[0..KEY_LENGTH_AES_USIZE].clone_from_slice(derived_left_bits);
@@ -101,24 +158,22 @@ impl EncryptedHashMap<H128> for SecretStore {
 	fn get<Value: FromRawBytes + BytesConvertable>(&self, key: &H128, password: &str) -> Result<Value, EncryptedHashMapError> {
 		match self.directory.get(key) {
 			Some(key_file) => {
-				let decrypted_bytes = match key_file.crypto.kdf {
+				let (derived_left_bits, derived_right_bits) = match key_file.crypto.kdf {
-					KeyFileKdf::Pbkdf2(ref params) => {
+					KeyFileKdf::Pbkdf2(ref params) => derive_key_iterations(password, &params.salt, params.c),
-						let (derived_left_bits, derived_right_bits) = derive_key_iterations(password, &params.salt, params.c);
+					KeyFileKdf::Scrypt(ref params) => derive_key_scrypt(password, &params.salt, params.n, params.p, params.r)
-						if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text)
-							.sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); }
-
-						let mut val = vec![0u8; key_file.crypto.cipher_text.len()];
-						match key_file.crypto.cipher_type {
-							CryptoCipherType::Aes128Ctr(ref iv) => {
-								crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val);
-							}
-						}
-						val
-					}
-					_ => { unimplemented!(); }
 				};
 
-				match Value::from_bytes(&decrypted_bytes) {
+				if derive_mac(&derived_right_bits, &key_file.crypto.cipher_text)
+					.sha3() != key_file.crypto.mac { return Err(EncryptedHashMapError::InvalidPassword); }
+
+				let mut val = vec![0u8; key_file.crypto.cipher_text.len()];
+				match key_file.crypto.cipher_type {
+					CryptoCipherType::Aes128Ctr(ref iv) => {
+						crypto::aes::decrypt(&derived_left_bits, &iv.as_slice(), &key_file.crypto.cipher_text, &mut val);
+					}
+				};
+
+				match Value::from_bytes(&val) {
 					Ok(value) => Ok(value),
 					Err(bytes_error) => Err(EncryptedHashMapError::InvalidValueFormat(bytes_error))
 				}
@@ -259,6 +314,27 @@ mod tests {
 		result
 	}
 
+	fn pregenerate_accounts(temp: &RandomTempPath, count: usize) -> Vec<H128> {
+		use keys::directory::{KeyFileContent, KeyFileCrypto};
+		let mut write_sstore = SecretStore::new_test(&temp);
+		let mut result = Vec::new();
+		for i in 0..count {
+			let mut key_file =
+				KeyFileContent::new(
+					KeyFileCrypto::new_pbkdf2(
+						FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
+						H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
+						H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
+						H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
+						262144,
+						32));
+			key_file.account = Some(x!(i as u64));
+			result.push(key_file.id.clone());
+			write_sstore.import_key(key_file).unwrap();
+		}
+		result
+	}
+
 	#[test]
 	fn can_get() {
 		let temp = RandomTempPath::create_dir();
@@ -293,5 +369,35 @@ mod tests {
 		assert_eq!(4, sstore.directory.list().unwrap().len())
 	}
 
+	#[test]
+	fn can_import_account() {
+		use keys::directory::{KeyFileContent, KeyFileCrypto};
+		let temp = RandomTempPath::create_dir();
+		let mut key_file =
+			KeyFileContent::new(
+				KeyFileCrypto::new_pbkdf2(
+					FromHex::from_hex("5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46").unwrap(),
+					H128::from_str("6087dab2f9fdbbfaddc31a909735c1e6").unwrap(),
+					H256::from_str("ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd").unwrap(),
+					H256::from_str("517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e9b2").unwrap(),
+					262144,
+					32));
+		key_file.account = Some(Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap());
 
+		let mut sstore = SecretStore::new_test(&temp);
+
+		sstore.import_key(key_file).unwrap();
+
+		assert_eq!(1, sstore.accounts().unwrap().len());
+		assert!(sstore.account(&Address::from_str("3f49624084b67849c7b4e805c5988c21a430f9d9").unwrap()).is_some());
+	}
+
+	#[test]
+	fn can_list_accounts() {
+		let temp = RandomTempPath::create_dir();
+		pregenerate_accounts(&temp, 30);
+		let sstore = SecretStore::new_test(&temp);
+		let accounts = sstore.accounts().unwrap();
+		assert_eq!(30, accounts.len());
+	}
 }