Pyethereum keystore support (#9710)

* support for keystore format produced by pyethereum lib + some debug msgs

* made salt unbound also for Scrypt

* made address optional and skip if its none

* ignore values of any type, not just i32

* WIP: local deps

* cleanup

* enable optional address + more cleanup

* yet another cleanup

* enable keystore wo address import (using passwd)

* cleanup

* doc enchancement

* parity/account fix

* redesign after review

* fix indentation

* ignore just version in json under crypto

* remove unnecessary borrowing within match str

Co-Authored-By: tworec <tworec@golem.network>

* remove another unnecessary borrowing

Co-Authored-By: tworec <tworec@golem.network>

* default derived instead of implementing

* log dependency removed

* [ethstore] warn restored + env_logger initialized in CLI

* applied suggestion from @tomusdrw

Cargo.lock

@@ -708,7 +708,7 @@ dependencies = [
  "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethjson 0.1.0",
  "ethkey 0.3.0",
- "ethstore 0.2.0",
+ "ethstore 0.2.1",
  "evm 0.1.0",
  "fake-hardware-wallet 0.0.1",
  "hardware-wallet 1.12.0",
@@ -1143,7 +1143,7 @@ dependencies = [
 
 [[package]]
 name = "ethstore"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
  "dir 0.1.2",
  "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1168,11 +1168,12 @@ dependencies = [
 
 [[package]]
 name = "ethstore-cli"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "dir 0.1.2",
  "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "ethstore 0.2.0",
+ "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ethstore 0.2.1",
  "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "panic_hook 0.1.0",
  "parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2526,7 +2527,7 @@ dependencies = [
  "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethjson 0.1.0",
  "ethkey 0.3.0",
- "ethstore 0.2.0",
+ "ethstore 0.2.1",
  "fake-fetch 0.0.1",
  "fake-hardware-wallet 0.0.1",
  "fastmap 0.1.0",

accounts/ethkey/cli/src/main.rs

@@ -168,7 +168,7 @@ fn main() {
 		Ok(ok) => println!("{}", ok),
 		Err(Error::Docopt(ref e)) => e.exit(),
 		Err(err) => {
-			println!("{}", err);
+			eprintln!("{}", err);
 			process::exit(1);
 		}
 	}

accounts/ethstore/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ethstore"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]

accounts/ethstore/cli/Cargo.toml

@@ -1,10 +1,11 @@
 [package]
 name = "ethstore-cli"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 docopt = "1.0"
+env_logger = "0.5"
 num_cpus = "1.6"
 rustc-hex = "1.0"
 serde = "1.0"

accounts/ethstore/cli/src/main.rs

@@ -23,6 +23,8 @@ extern crate parking_lot;
 extern crate rustc_hex;
 extern crate serde;
 
+extern crate env_logger;
+
 #[macro_use]
 extern crate serde_derive;
 
@@ -45,7 +47,7 @@ Usage:
     ethstore insert <secret> <password> [--dir DIR] [--vault VAULT] [--vault-pwd VAULTPWD]
     ethstore change-pwd <address> <old-pwd> <new-pwd> [--dir DIR] [--vault VAULT] [--vault-pwd VAULTPWD]
     ethstore list [--dir DIR] [--vault VAULT] [--vault-pwd VAULTPWD]
-    ethstore import [--src DIR] [--dir DIR]
+    ethstore import [<password>] [--src DIR] [--dir DIR]
     ethstore import-wallet <path> <password> [--dir DIR] [--vault VAULT] [--vault-pwd VAULTPWD]
     ethstore find-wallet-pass <path> <password>
     ethstore remove <address> <password> [--dir DIR] [--vault VAULT] [--vault-pwd VAULTPWD]
@@ -146,30 +148,34 @@ impl fmt::Display for Error {
 
 fn main() {
 	panic_hook::set_abort();
+	if env::var("RUST_LOG").is_err() {
+		env::set_var("RUST_LOG", "warn")
+	}
+	env_logger::try_init().expect("Logger initialized only once.");
 
 	match execute(env::args()) {
 		Ok(result) => println!("{}", result),
 		Err(Error::Docopt(ref e)) => e.exit(),
 		Err(err) => {
-			println!("{}", err);
+			eprintln!("{}", err);
 			process::exit(1);
 		}
 	}
 }
 
-fn key_dir(location: &str) -> Result<Box<KeyDirectory>, Error> {
+fn key_dir(location: &str, password: Option<Password>) -> Result<Box<KeyDirectory>, Error> {
-	let dir: Box<KeyDirectory> = match location {
+	let dir: RootDiskDirectory = match location {
-		"geth" => Box::new(RootDiskDirectory::create(dir::geth(false))?),
+		"geth" => RootDiskDirectory::create(dir::geth(false))?,
-		"geth-test" => Box::new(RootDiskDirectory::create(dir::geth(true))?),
+		"geth-test" => RootDiskDirectory::create(dir::geth(true))?,
 		path if path.starts_with("parity") => {
 			let chain = path.split('-').nth(1).unwrap_or("ethereum");
 			let path = dir::parity(chain);
-			Box::new(RootDiskDirectory::create(path)?)
+			RootDiskDirectory::create(path)?
 		},
-		path => Box::new(RootDiskDirectory::create(path)?),
+		path => RootDiskDirectory::create(path)?,
 	};
 
-	Ok(dir)
+	Ok(Box::new(dir.with_password(password)))
 }
 
 fn open_args_vault(store: &EthStore, args: &Args) -> Result<SecretVaultRef, Error> {
@@ -202,9 +208,9 @@ fn format_vaults(vaults: &[String]) -> String {
 }
 
 fn load_password(path: &str) -> Result<Password, Error> {
-	let mut file = fs::File::open(path).map_err(|e| ethstore::Error::Custom(format!("Error opening password file {}: {}", path, e)))?;
+	let mut file = fs::File::open(path).map_err(|e| ethstore::Error::Custom(format!("Error opening password file '{}': {}", path, e)))?;
 	let mut password = String::new();
-	file.read_to_string(&mut password).map_err(|e| ethstore::Error::Custom(format!("Error reading password file {}: {}", path, e)))?;
+	file.read_to_string(&mut password).map_err(|e| ethstore::Error::Custom(format!("Error reading password file '{}': {}", path, e)))?;
 	// drop EOF
 	let _ = password.pop();
 	Ok(password.into())
@@ -214,7 +220,7 @@ fn execute<S, I>(command: I) -> Result<String, Error> where I: IntoIterator<Item
 	let args: Args = Docopt::new(USAGE)
 		.and_then(|d| d.argv(command).deserialize())?;
 
-	let store = EthStore::open(key_dir(&args.flag_dir)?)?;
+	let store = EthStore::open(key_dir(&args.flag_dir, None)?)?;
 
 	return if args.cmd_insert {
 		let secret = args.arg_secret.parse().map_err(|_| ethstore::Error::InvalidSecret)?;
@@ -239,8 +245,13 @@ fn execute<S, I>(command: I) -> Result<String, Error> where I: IntoIterator<Item
 			.collect();
 		Ok(format_accounts(&accounts))
 	} else if args.cmd_import {
-		let src = key_dir(&args.flag_src)?;
+		let password = match args.arg_password.as_ref() {
-		let dst = key_dir(&args.flag_dir)?;
+			"" => None,
+			_ => Some(load_password(&args.arg_password)?)
+		};
+		let src = key_dir(&args.flag_src, password)?;
+		let dst = key_dir(&args.flag_dir, None)?;
+
 		let accounts = import_accounts(&*src, &*dst)?;
 		Ok(format_accounts(&accounts))
 	} else if args.cmd_import_wallet {

accounts/ethstore/src/account/crypto.rs

@@ -84,7 +84,8 @@ impl Crypto {
 
 		// two parts of derived key
 		// DK = [ DK[0..15] DK[16..31] ] = [derived_left_bits, derived_right_bits]
-		let (derived_left_bits, derived_right_bits) = crypto::derive_key_iterations(password.as_bytes(), &salt, iterations);
+		let (derived_left_bits, derived_right_bits) =
+			crypto::derive_key_iterations(password.as_bytes(), &salt, iterations);
 
 		// preallocated (on-stack in case of `Secret`) buffer to hold cipher
 		// length = length(plain) as we are using CTR-approach
@@ -104,7 +105,7 @@ impl Crypto {
 			ciphertext: ciphertext.into_vec(),
 			kdf: Kdf::Pbkdf2(Pbkdf2 {
 				dklen: crypto::KEY_LENGTH as u32,
-				salt: salt,
+				salt: salt.to_vec(),
 				c: iterations,
 				prf: Prf::HmacSha256,
 			}),

accounts/ethstore/src/account/kdf.rs

@@ -26,7 +26,7 @@ pub struct Pbkdf2 {
 	pub c: u32,
 	pub dklen: u32,
 	pub prf: Prf,
-	pub salt: [u8; 32],
+	pub salt: Vec<u8>,
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -35,7 +35,7 @@ pub struct Scrypt {
 	pub p: u32,
 	pub n: u32,
 	pub r: u32,
-	pub salt: [u8; 32],
+	pub salt: Vec<u8>,
 }
 
 #[derive(Debug, PartialEq, Clone)]

accounts/ethstore/src/account/safe_account.rs

@@ -45,7 +45,7 @@ impl Into<json::KeyFile> for SafeAccount {
 		json::KeyFile {
 			id: From::from(self.id),
 			version: self.version.into(),
-			address: self.address.into(),
+			address: Some(self.address.into()),
 			crypto: self.crypto.into(),
 			name: Some(self.name.into()),
 			meta: Some(self.meta.into()),
@@ -77,16 +77,43 @@ impl SafeAccount {
 	/// Create a new `SafeAccount` from the given `json`; if it was read from a
 	/// file, the `filename` should be `Some` name. If it is as yet anonymous, then it
 	/// can be left `None`.
-	pub fn from_file(json: json::KeyFile, filename: Option<String>) -> Self {
+	/// In case `password` is provided, we will attempt to read the secret from the keyfile
-		SafeAccount {
+	/// and derive the address from it instead of reading it directly.
+	/// Providing password is required for `json::KeyFile`s with no address.
+	pub fn from_file(json: json::KeyFile, filename: Option<String>, password: &Option<Password>) -> Result<Self, Error> {
+		let crypto = Crypto::from(json.crypto);
+		let address = match (password, &json.address) {
+			(None, Some(json_address)) => json_address.into(),
+			(None, None) => Err(Error::Custom(
+				"This keystore does not contain address. You need to provide password to import it".into()))?,
+			(Some(password), json_address) => {
+				let derived_address = KeyPair::from_secret(
+					crypto.secret(&password).map_err(|_| Error::InvalidPassword)?
+				)?.address();
+
+				match json_address {
+					Some(json_address) => {
+						let json_address = json_address.into();
+						if derived_address != json_address {
+							warn!("Detected address mismatch when opening an account. Derived: {:?}, in json got: {:?}",
+								derived_address, json_address);
+						}
+					},
+					_ => {},
+				}
+				derived_address
+			}
+		};
+
+		Ok(SafeAccount {
 			id: json.id.into(),
 			version: json.version.into(),
-			address: json.address.into(),
+			address,
-			crypto: json.crypto.into(),
+			crypto,
-			filename: filename,
+			filename,
 			name: json.name.unwrap_or(String::new()),
 			meta: json.meta.unwrap_or("{}".to_owned()),
-		}
+		})
 	}
 
 	/// Create a new `SafeAccount` from the given vault `json`; if it was read from a
@@ -97,14 +124,14 @@ impl SafeAccount {
 		let meta_plain = meta_crypto.decrypt(password)?;
 		let meta_plain = json::VaultKeyMeta::load(&meta_plain).map_err(|e| Error::Custom(format!("{:?}", e)))?;
 
-		Ok(SafeAccount::from_file(json::KeyFile {
+		SafeAccount::from_file(json::KeyFile {
 			id: json.id,
 			version: json.version,
 			crypto: json.crypto,
-			address: meta_plain.address,
+			address: Some(meta_plain.address),
 			name: meta_plain.name,
 			meta: meta_plain.meta,
-		}, filename))
+		}, filename, &None)
 	}
 
 	/// Create a new `VaultKeyFile` from the given `self`

accounts/ethstore/src/accounts_dir/disk.rs

@@ -23,6 +23,7 @@ use {json, SafeAccount, Error};
 use json::Uuid;
 use super::{KeyDirectory, VaultKeyDirectory, VaultKeyDirectoryProvider, VaultKey};
 use super::vault::{VAULT_FILE_NAME, VaultDiskDirectory};
+use ethkey::Password;
 
 const IGNORED_FILES: &'static [&'static str] = &[
 	"thumbs.db",
@@ -106,6 +107,7 @@ pub type RootDiskDirectory = DiskDirectory<DiskKeyFileManager>;
 pub trait KeyFileManager: Send + Sync {
 	/// Read `SafeAccount` from given key file stream
 	fn read<T>(&self, filename: Option<String>, reader: T) -> Result<SafeAccount, Error> where T: io::Read;
+
 	/// Write `SafeAccount` to given key file stream
 	fn write<T>(&self, account: SafeAccount, writer: &mut T) -> Result<(), Error> where T: io::Write;
 }
@@ -117,7 +119,10 @@ pub struct DiskDirectory<T> where T: KeyFileManager {
 }
 
 /// Keys file manager for root keys directory
-pub struct DiskKeyFileManager;
+#[derive(Default)]
+pub struct DiskKeyFileManager {
+	password: Option<Password>,
+}
 
 impl RootDiskDirectory {
 	pub fn create<P>(path: P) -> Result<Self, Error> where P: AsRef<Path> {
@@ -125,8 +130,14 @@ impl RootDiskDirectory {
 		Ok(Self::at(path))
 	}
 
+	/// allows to read keyfiles with given password (needed for keyfiles w/o address)
+	pub fn with_password(&self, password: Option<Password>) -> Self {
+		DiskDirectory::new(&self.path, DiskKeyFileManager { password })
+	}
+
+
 	pub fn at<P>(path: P) -> Self where P: AsRef<Path> {
-		DiskDirectory::new(path, DiskKeyFileManager)
+		DiskDirectory::new(path, DiskKeyFileManager::default())
 	}
 }
 
@@ -319,7 +330,7 @@ impl<T> VaultKeyDirectoryProvider for DiskDirectory<T> where T: KeyFileManager {
 impl KeyFileManager for DiskKeyFileManager {
 	fn read<T>(&self, filename: Option<String>, reader: T) -> Result<SafeAccount, Error> where T: io::Read {
 		let key_file = json::KeyFile::load(reader).map_err(|e| Error::Custom(format!("{:?}", e)))?;
-		Ok(SafeAccount::from_file(key_file, filename))
+		SafeAccount::from_file(key_file, filename, &self.password)
 	}
 
 	fn write<T>(&self, mut account: SafeAccount, writer: &mut T) -> Result<(), Error> where T: io::Write {

accounts/ethstore/src/ethstore.rs

@@ -168,7 +168,7 @@ impl SecretStore for EthStore {
 
 	fn import_wallet(&self, vault: SecretVaultRef, json: &[u8], password: &Password, gen_id: bool) -> Result<StoreAccountRef, Error> {
 		let json_keyfile = json::KeyFile::load(json).map_err(|_| Error::InvalidKeyFile("Invalid JSON format".to_owned()))?;
-		let mut safe_account = SafeAccount::from_file(json_keyfile, None);
+		let mut safe_account = SafeAccount::from_file(json_keyfile, None, &None)?;
 
 		if gen_id {
 			safe_account.id = Random::random();

accounts/ethstore/src/import.rs

@@ -25,7 +25,7 @@ use Error;
 
 /// Import an account from a file.
 pub fn import_account(path: &Path, dst: &KeyDirectory) -> Result<Address, Error> {
-	let key_manager = DiskKeyFileManager;
+	let key_manager = DiskKeyFileManager::default();
 	let existing_accounts = dst.load()?.into_iter().map(|a| a.address).collect::<HashSet<_>>();
 	let filename = path.file_name().and_then(|n| n.to_str()).map(|f| f.to_owned());
 	let account = fs::File::open(&path)
@@ -42,7 +42,9 @@ pub fn import_account(path: &Path, dst: &KeyDirectory) -> Result<Address, Error>
 /// Import all accounts from one directory to the other.
 pub fn import_accounts(src: &KeyDirectory, dst: &KeyDirectory) -> Result<Vec<Address>, Error> {
 	let accounts = src.load()?;
-	let existing_accounts = dst.load()?.into_iter().map(|a| a.address).collect::<HashSet<_>>();
+	let existing_accounts = dst.load()?.into_iter()
+		.map(|a| a.address)
+		.collect::<HashSet<_>>();
 
 	accounts.into_iter()
 		.filter(|a| !existing_accounts.contains(&a.address))

accounts/ethstore/src/json/crypto.rs

@@ -52,6 +52,7 @@ enum CryptoField {
 	Kdf,
 	KdfParams,
 	Mac,
+	Version,
 }
 
 impl<'a> Deserialize<'a> for CryptoField {
@@ -81,6 +82,7 @@ impl<'a> Visitor<'a> for CryptoFieldVisitor {
 			"kdf" => Ok(CryptoField::Kdf),
 			"kdfparams" => Ok(CryptoField::KdfParams),
 			"mac" => Ok(CryptoField::Mac),
+			"version" => Ok(CryptoField::Version),
 			_ => Err(Error::custom(format!("Unknown field: '{}'", value))),
 		}
 	}
@@ -122,6 +124,8 @@ impl<'a> Visitor<'a> for CryptoVisitor {
 				Some(CryptoField::Kdf) => { kdf = Some(visitor.next_value()?); }
 				Some(CryptoField::KdfParams) => { kdfparams = Some(visitor.next_value()?); }
 				Some(CryptoField::Mac) => { mac = Some(visitor.next_value()?); }
+				// skip not required version field (it appears in pyethereum generated keystores)
+				Some(CryptoField::Version) => { visitor.next_value().unwrap_or(()) }
 				None => { break; }
 			}
 		}

accounts/ethstore/src/json/kdf.rs

@@ -17,7 +17,7 @@
 use std::fmt;
 use serde::{Serialize, Serializer, Deserialize, Deserializer};
 use serde::de::{Visitor, Error as SerdeError};
-use super::{Error, H256};
+use super::{Error, Bytes};
 
 #[derive(Debug, PartialEq)]
 pub enum KdfSer {
@@ -111,7 +111,7 @@ pub struct Pbkdf2 {
 	pub c: u32,
 	pub dklen: u32,
 	pub prf: Prf,
-	pub salt: H256,
+	pub salt: Bytes,
 }
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
@@ -120,7 +120,7 @@ pub struct Scrypt {
 	pub p: u32,
 	pub n: u32,
 	pub r: u32,
-	pub salt: H256,
+	pub salt: Bytes,
 }
 
 #[derive(Debug, PartialEq)]

accounts/ethstore/src/json/key_file.rs

@@ -46,7 +46,7 @@ pub struct KeyFile {
 	pub id: Uuid,
 	pub version: Version,
 	pub crypto: Crypto,
-	pub address: H160,
+	pub address: Option<H160>,
 	pub name: Option<String>,
 	pub meta: Option<String>,
 }
@@ -158,11 +158,6 @@ impl<'a> Visitor<'a> for KeyFileVisitor {
 			None => return Err(V::Error::missing_field("crypto")),
 		};
 
-		let address = match address {
-			Some(address) => address,
-			None => return Err(V::Error::missing_field("address")),
-		};
-
 		let result = KeyFile {
 			id: id,
 			version: version,
@@ -222,7 +217,7 @@ mod tests {
 		let expected = KeyFile {
 			id: Uuid::from_str("8777d9f6-7860-4b9b-88b7-0b57ee6b3a73").unwrap(),
 			version: Version::V3,
-			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
+			address: Some("6edddfc6349aff20bc6467ccf276c5b52487f7a8".into()),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
 					iv: "b5a7ec855ec9e2c405371356855fec83".into(),
@@ -273,7 +268,7 @@ mod tests {
 		let expected = KeyFile {
 			id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(),
 			version: Version::V3,
-			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
+			address: Some("6edddfc6349aff20bc6467ccf276c5b52487f7a8".into()),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
 					iv: "b5a7ec855ec9e2c405371356855fec83".into(),
@@ -301,7 +296,7 @@ mod tests {
 		let file = KeyFile {
 			id: "8777d9f6-7860-4b9b-88b7-0b57ee6b3a73".into(),
 			version: Version::V3,
-			address: "6edddfc6349aff20bc6467ccf276c5b52487f7a8".into(),
+			address: Some("6edddfc6349aff20bc6467ccf276c5b52487f7a8".into()),
 			crypto: Crypto {
 				cipher: Cipher::Aes128Ctr(Aes128Ctr {
 					iv: "b5a7ec855ec9e2c405371356855fec83".into(),