From d4149b965e87e701da0334f7f15eb0e48707a5c3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 16 Feb 2017 18:20:24 +0300 Subject: [PATCH 1/5] files list separate fn, sha3 of the list --- ethstore/src/dir/disk.rs | 28 ++++++++++++++++------------ util/src/sha3.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/ethstore/src/dir/disk.rs b/ethstore/src/dir/disk.rs index 4a4637850..023db1ac2 100755 --- a/ethstore/src/dir/disk.rs +++ b/ethstore/src/dir/disk.rs @@ -90,11 +90,8 @@ impl DiskDirectory where T: KeyFileManager { } } - /// all accounts found in keys directory - fn files(&self) -> Result, Error> { - // it's not done using one iterator cause - // there is an issue with rustc and it takes tooo much time to compile - let paths = fs::read_dir(&self.path)? + fn files(&self) -> Result, Error> { + Ok(fs::read_dir(&self.path)? .flat_map(Result::ok) .filter(|entry| { let metadata = entry.metadata().ok(); @@ -102,14 +99,21 @@ impl DiskDirectory where T: KeyFileManager { let name = file_name.to_string_lossy(); // filter directories metadata.map_or(false, |m| !m.is_dir()) && - // hidden files - !name.starts_with(".") && - // other ignored files - !IGNORED_FILES.contains(&&*name) + // hidden files + !name.starts_with(".") && + // other ignored files + !IGNORED_FILES.contains(&&*name) }) .map(|entry| entry.path()) - .collect::>(); + .collect::>() + ) + } + /// all accounts found in keys directory + fn files_content(&self) -> Result, Error> { + // it's not done using one iterator cause + // there is an issue with rustc and it takes tooo much time to compile + let paths = self.files()?; Ok(paths .into_iter() .filter_map(|path| { @@ -166,7 +170,7 @@ impl DiskDirectory where T: KeyFileManager { impl KeyDirectory for DiskDirectory where T: KeyFileManager { fn load(&self) -> Result, Error> { - let accounts = self.files()? + let accounts = self.files_content()? .into_iter() .map(|(_, account)| account) .collect(); @@ -191,7 +195,7 @@ impl KeyDirectory for DiskDirectory where T: KeyFileManager { fn remove(&self, account: &SafeAccount) -> Result<(), Error> { // enumerate all entries in keystore // and find entry with given address - let to_remove = self.files()? + let to_remove = self.files_content()? .into_iter() .find(|&(_, ref acc)| acc.id == account.id && acc.address == account.address); diff --git a/util/src/sha3.rs b/util/src/sha3.rs index 45d6a34d5..4d3502313 100644 --- a/util/src/sha3.rs +++ b/util/src/sha3.rs @@ -68,6 +68,24 @@ impl Hashable for T where T: AsRef<[u8]> { } } +impl Hashable for [T] where T: Hashable { + fn sha3(&self) -> H256 { + use std::ops::BitXor; + + let mut sha3 = SHA3_EMPTY; + for t in self.iter() { + sha3 = sha3.bitxor(t.sha3()); + }; + + sha3 + } + // todo: optimize? + fn sha3_into(&self, dest: &mut [u8]) { + let sha3 = self.sha3(); + dest.copy_from_slice(&*sha3); + } +} + /// Calculate SHA3 of given stream. pub fn sha3(r: &mut io::BufRead) -> Result { let mut output = [0u8; 32]; @@ -120,4 +138,15 @@ mod tests { // then assert_eq!(format!("{:?}", hash), "68371d7e884c168ae2022c82bd837d51837718a7f7dfb7aa3f753074a35e1d87"); } + + #[test] + fn should_sha3_strs() { + let strs = vec!["abc".to_owned(), "gdc".to_owned()]; + let hash = strs.sha3(); + assert_eq!(hash, "b8f24d705171c55892a34c7b863c258f4d47e6864f7a7da45f84155597a3b338".parse().unwrap()); + + let strs = vec!["abc".to_owned(), "gdc_".to_owned()]; + let hash = strs.sha3(); + assert_eq!(hash, "41bd661b8e02faccad55cdbb28db974dd5c9ae41825b89907fcf25db793b8b09".parse().unwrap()); + } } From 43ce5bef7e156b9560d6e5ad6a24992e6deaf4cf Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 16 Feb 2017 18:47:58 +0300 Subject: [PATCH 2/5] file list hash and test --- ethstore/src/dir/disk.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/ethstore/src/dir/disk.rs b/ethstore/src/dir/disk.rs index 023db1ac2..c8afcc910 100755 --- a/ethstore/src/dir/disk.rs +++ b/ethstore/src/dir/disk.rs @@ -22,6 +22,7 @@ use {json, SafeAccount, Error}; use json::Uuid; use super::{KeyDirectory, VaultKeyDirectory, VaultKeyDirectoryProvider, VaultKey}; use super::vault::{VAULT_FILE_NAME, VaultDiskDirectory}; +use util::H256; const IGNORED_FILES: &'static [&'static str] = &[ "thumbs.db", @@ -109,6 +110,13 @@ impl DiskDirectory where T: KeyFileManager { ) } + pub fn files_hash(&self) -> Result { + use util::Hashable; + let files = self.files()?; + let file_strs: Vec<&str> = files.iter().map(|fp| fp.to_str().unwrap_or("")).collect(); + Ok(file_strs.sha3()) + } + /// all accounts found in keys directory fn files_content(&self) -> Result, Error> { // it's not done using one iterator cause @@ -283,7 +291,6 @@ mod test { let account = SafeAccount::create(&keypair, [0u8; 16], password, 1024, "Test".to_owned(), "{}".to_owned()); let res = directory.insert(account); - // then assert!(res.is_ok(), "Should save account succesfuly."); assert!(res.unwrap().filename.is_some(), "Filename has been assigned."); @@ -340,4 +347,25 @@ mod test { assert!(vaults.iter().any(|v| &*v == "vault1")); assert!(vaults.iter().any(|v| &*v == "vault2")); } + + #[test] + fn hash_of_files() { + let temp_path = RandomTempPath::new(); + let directory = RootDiskDirectory::create(&temp_path).unwrap(); + + let hash = directory.files_hash().expect("Files hash should be calculated ok"); + assert_eq!( + hash, + "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".parse().unwrap() + ); + + let keypair = Random.generate().unwrap(); + let password = "test pass"; + let account = SafeAccount::create(&keypair, [0u8; 16], password, 1024, "Test".to_owned(), "{}".to_owned()); + directory.insert(account).expect("Account should be inserted ok"); + + let new_hash = directory.files_hash().expect("New files hash should be calculated ok"); + + assert!(new_hash != hash, "hash of the file list should change once directory content changed"); + } } From 513cc6261ad3f9fa9337e6196c4f11c708d3cb82 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 16 Feb 2017 22:10:29 +0300 Subject: [PATCH 3/5] plug to store --- ethstore/src/dir/disk.rs | 4 ++++ ethstore/src/dir/mod.rs | 3 +++ ethstore/src/ethstore.rs | 20 +++++++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ethstore/src/dir/disk.rs b/ethstore/src/dir/disk.rs index c8afcc910..ccc84077b 100755 --- a/ethstore/src/dir/disk.rs +++ b/ethstore/src/dir/disk.rs @@ -219,6 +219,10 @@ impl KeyDirectory for DiskDirectory where T: KeyFileManager { fn as_vault_provider(&self) -> Option<&VaultKeyDirectoryProvider> { Some(self) } + + fn hash(&self) -> Result { + self.files_hash() + } } impl VaultKeyDirectoryProvider for DiskDirectory where T: KeyFileManager { diff --git a/ethstore/src/dir/mod.rs b/ethstore/src/dir/mod.rs index 6e4326968..11a1a2f23 100755 --- a/ethstore/src/dir/mod.rs +++ b/ethstore/src/dir/mod.rs @@ -16,6 +16,7 @@ use std::path::{PathBuf}; use {SafeAccount, Error}; +use util::{H256, FixedHash}; mod disk; mod geth; @@ -62,6 +63,8 @@ pub trait KeyDirectory: Send + Sync { fn path(&self) -> Option<&PathBuf> { None } /// Return vault provider, if available fn as_vault_provider(&self) -> Option<&VaultKeyDirectoryProvider> { None } + /// Returns hash of the directory content, if supported + fn hash(&self) -> Result { Ok(H256::zero()) } } /// Vaults provider diff --git a/ethstore/src/ethstore.rs b/ethstore/src/ethstore.rs index 86b6dce46..eecc361e4 100755 --- a/ethstore/src/ethstore.rs +++ b/ethstore/src/ethstore.rs @@ -27,6 +27,7 @@ use account::SafeAccount; use presale::PresaleWallet; use json::{self, Uuid}; use {import, Error, SimpleSecretStore, SecretStore, SecretVaultRef, StoreAccountRef, Derivation}; +use util::H256; pub struct EthStore { store: EthMultiStore, @@ -230,6 +231,7 @@ pub struct EthMultiStore { // order lock: cache, then vaults cache: RwLock>>, vaults: Mutex>>, + dir_hash: RwLock>, } impl EthMultiStore { @@ -244,11 +246,23 @@ impl EthMultiStore { vaults: Mutex::new(HashMap::new()), iterations: iterations, cache: Default::default(), + dir_hash: Default::default(), }; store.reload_accounts()?; Ok(store) } + fn reload_if_changed(&self) -> Result<(), Error> { + let mut last_dir_hash = self.dir_hash.write(); + let dir_hash = Some(self.dir.hash()?); + if *last_dir_hash == dir_hash { + return Ok(()) + } + self.reload_accounts()?; + *last_dir_hash = dir_hash; + Ok(()) + } + fn reload_accounts(&self) -> Result<(), Error> { let mut cache = self.cache.write(); @@ -284,7 +298,7 @@ impl EthMultiStore { } } - self.reload_accounts()?; + self.reload_if_changed()?; let cache = self.cache.read(); let accounts = cache.get(account).ok_or(Error::InvalidAccount)?; if accounts.is_empty() { @@ -431,7 +445,7 @@ impl SimpleSecretStore for EthMultiStore { } fn account_ref(&self, address: &Address) -> Result { - self.reload_accounts()?; + self.reload_if_changed()?; self.cache.read().keys() .find(|r| &r.address == address) .cloned() @@ -439,7 +453,7 @@ impl SimpleSecretStore for EthMultiStore { } fn accounts(&self) -> Result, Error> { - self.reload_accounts()?; + self.reload_if_changed()?; Ok(self.cache.read().keys().cloned().collect()) } From 444065e2942e0963b5ee8f3b584066b08c843d98 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 16 Feb 2017 22:53:58 +0300 Subject: [PATCH 4/5] refactor hashing --- ethstore/src/dir/disk.rs | 19 ++++++++++++------- ethstore/src/dir/mod.rs | 3 +-- ethstore/src/ethstore.rs | 5 ++--- util/src/sha3.rs | 29 ----------------------------- 4 files changed, 15 insertions(+), 41 deletions(-) diff --git a/ethstore/src/dir/disk.rs b/ethstore/src/dir/disk.rs index ccc84077b..f8d8345e7 100755 --- a/ethstore/src/dir/disk.rs +++ b/ethstore/src/dir/disk.rs @@ -22,7 +22,6 @@ use {json, SafeAccount, Error}; use json::Uuid; use super::{KeyDirectory, VaultKeyDirectory, VaultKeyDirectoryProvider, VaultKey}; use super::vault::{VAULT_FILE_NAME, VaultDiskDirectory}; -use util::H256; const IGNORED_FILES: &'static [&'static str] = &[ "thumbs.db", @@ -110,11 +109,17 @@ impl DiskDirectory where T: KeyFileManager { ) } - pub fn files_hash(&self) -> Result { - use util::Hashable; + pub fn files_hash(&self) -> Result { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + + let mut hasher = DefaultHasher::new(); let files = self.files()?; - let file_strs: Vec<&str> = files.iter().map(|fp| fp.to_str().unwrap_or("")).collect(); - Ok(file_strs.sha3()) + for file in files { + hasher.write(file.to_str().unwrap_or("").as_bytes()) + } + + Ok(hasher.finish()) } /// all accounts found in keys directory @@ -220,7 +225,7 @@ impl KeyDirectory for DiskDirectory where T: KeyFileManager { Some(self) } - fn hash(&self) -> Result { + fn hash(&self) -> Result { self.files_hash() } } @@ -360,7 +365,7 @@ mod test { let hash = directory.files_hash().expect("Files hash should be calculated ok"); assert_eq!( hash, - "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".parse().unwrap() + 15130871412783076140 ); let keypair = Random.generate().unwrap(); diff --git a/ethstore/src/dir/mod.rs b/ethstore/src/dir/mod.rs index 11a1a2f23..5fbf5fb8b 100755 --- a/ethstore/src/dir/mod.rs +++ b/ethstore/src/dir/mod.rs @@ -16,7 +16,6 @@ use std::path::{PathBuf}; use {SafeAccount, Error}; -use util::{H256, FixedHash}; mod disk; mod geth; @@ -64,7 +63,7 @@ pub trait KeyDirectory: Send + Sync { /// Return vault provider, if available fn as_vault_provider(&self) -> Option<&VaultKeyDirectoryProvider> { None } /// Returns hash of the directory content, if supported - fn hash(&self) -> Result { Ok(H256::zero()) } + fn hash(&self) -> Result { Ok(0u64) } } /// Vaults provider diff --git a/ethstore/src/ethstore.rs b/ethstore/src/ethstore.rs index eecc361e4..401630ba8 100755 --- a/ethstore/src/ethstore.rs +++ b/ethstore/src/ethstore.rs @@ -27,7 +27,6 @@ use account::SafeAccount; use presale::PresaleWallet; use json::{self, Uuid}; use {import, Error, SimpleSecretStore, SecretStore, SecretVaultRef, StoreAccountRef, Derivation}; -use util::H256; pub struct EthStore { store: EthMultiStore, @@ -231,7 +230,7 @@ pub struct EthMultiStore { // order lock: cache, then vaults cache: RwLock>>, vaults: Mutex>>, - dir_hash: RwLock>, + dir_hash: Mutex>, } impl EthMultiStore { @@ -253,7 +252,7 @@ impl EthMultiStore { } fn reload_if_changed(&self) -> Result<(), Error> { - let mut last_dir_hash = self.dir_hash.write(); + let mut last_dir_hash = self.dir_hash.lock(); let dir_hash = Some(self.dir.hash()?); if *last_dir_hash == dir_hash { return Ok(()) diff --git a/util/src/sha3.rs b/util/src/sha3.rs index 4d3502313..45d6a34d5 100644 --- a/util/src/sha3.rs +++ b/util/src/sha3.rs @@ -68,24 +68,6 @@ impl Hashable for T where T: AsRef<[u8]> { } } -impl Hashable for [T] where T: Hashable { - fn sha3(&self) -> H256 { - use std::ops::BitXor; - - let mut sha3 = SHA3_EMPTY; - for t in self.iter() { - sha3 = sha3.bitxor(t.sha3()); - }; - - sha3 - } - // todo: optimize? - fn sha3_into(&self, dest: &mut [u8]) { - let sha3 = self.sha3(); - dest.copy_from_slice(&*sha3); - } -} - /// Calculate SHA3 of given stream. pub fn sha3(r: &mut io::BufRead) -> Result { let mut output = [0u8; 32]; @@ -138,15 +120,4 @@ mod tests { // then assert_eq!(format!("{:?}", hash), "68371d7e884c168ae2022c82bd837d51837718a7f7dfb7aa3f753074a35e1d87"); } - - #[test] - fn should_sha3_strs() { - let strs = vec!["abc".to_owned(), "gdc".to_owned()]; - let hash = strs.sha3(); - assert_eq!(hash, "b8f24d705171c55892a34c7b863c258f4d47e6864f7a7da45f84155597a3b338".parse().unwrap()); - - let strs = vec!["abc".to_owned(), "gdc_".to_owned()]; - let hash = strs.sha3(); - assert_eq!(hash, "41bd661b8e02faccad55cdbb28db974dd5c9ae41825b89907fcf25db793b8b09".parse().unwrap()); - } } From 92d8edc1a6b06886cb0ed8eee274de158cfa772f Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 16 Feb 2017 23:04:39 +0300 Subject: [PATCH 5/5] unique_repr, no default impl --- ethstore/src/dir/disk.rs | 2 +- ethstore/src/dir/geth.rs | 4 ++++ ethstore/src/dir/memory.rs | 7 +++++++ ethstore/src/dir/mod.rs | 4 ++-- ethstore/src/dir/parity.rs | 4 ++++ ethstore/src/ethstore.rs | 2 +- ethstore/tests/util/transient_dir.rs | 4 ++++ 7 files changed, 23 insertions(+), 4 deletions(-) diff --git a/ethstore/src/dir/disk.rs b/ethstore/src/dir/disk.rs index f8d8345e7..afca0955d 100755 --- a/ethstore/src/dir/disk.rs +++ b/ethstore/src/dir/disk.rs @@ -225,7 +225,7 @@ impl KeyDirectory for DiskDirectory where T: KeyFileManager { Some(self) } - fn hash(&self) -> Result { + fn unique_repr(&self) -> Result { self.files_hash() } } diff --git a/ethstore/src/dir/geth.rs b/ethstore/src/dir/geth.rs index 1058e433f..0dfa2e6b2 100755 --- a/ethstore/src/dir/geth.rs +++ b/ethstore/src/dir/geth.rs @@ -95,4 +95,8 @@ impl KeyDirectory for GethDirectory { fn remove(&self, account: &SafeAccount) -> Result<(), Error> { self.dir.remove(account) } + + fn unique_repr(&self) -> Result { + self.dir.unique_repr() + } } diff --git a/ethstore/src/dir/memory.rs b/ethstore/src/dir/memory.rs index 941795efc..87d12794e 100644 --- a/ethstore/src/dir/memory.rs +++ b/ethstore/src/dir/memory.rs @@ -63,5 +63,12 @@ impl KeyDirectory for MemoryDirectory { } Ok(()) } + + fn unique_repr(&self) -> Result { + let mut val = 0u64; + let accounts = self.accounts.read(); + for acc in accounts.keys() { val = val ^ ::util::FixedHash::low_u64(acc) } + Ok(val) + } } diff --git a/ethstore/src/dir/mod.rs b/ethstore/src/dir/mod.rs index 5fbf5fb8b..83e978707 100755 --- a/ethstore/src/dir/mod.rs +++ b/ethstore/src/dir/mod.rs @@ -62,8 +62,8 @@ pub trait KeyDirectory: Send + Sync { fn path(&self) -> Option<&PathBuf> { None } /// Return vault provider, if available fn as_vault_provider(&self) -> Option<&VaultKeyDirectoryProvider> { None } - /// Returns hash of the directory content, if supported - fn hash(&self) -> Result { Ok(0u64) } + /// Unique representation of directory account collection + fn unique_repr(&self) -> Result; } /// Vaults provider diff --git a/ethstore/src/dir/parity.rs b/ethstore/src/dir/parity.rs index 198e50165..df03260d3 100755 --- a/ethstore/src/dir/parity.rs +++ b/ethstore/src/dir/parity.rs @@ -74,4 +74,8 @@ impl KeyDirectory for ParityDirectory { fn remove(&self, account: &SafeAccount) -> Result<(), Error> { self.dir.remove(account) } + + fn unique_repr(&self) -> Result { + self.dir.unique_repr() + } } diff --git a/ethstore/src/ethstore.rs b/ethstore/src/ethstore.rs index 401630ba8..3cdf0e643 100755 --- a/ethstore/src/ethstore.rs +++ b/ethstore/src/ethstore.rs @@ -253,7 +253,7 @@ impl EthMultiStore { fn reload_if_changed(&self) -> Result<(), Error> { let mut last_dir_hash = self.dir_hash.lock(); - let dir_hash = Some(self.dir.hash()?); + let dir_hash = Some(self.dir.unique_repr()?); if *last_dir_hash == dir_hash { return Ok(()) } diff --git a/ethstore/tests/util/transient_dir.rs b/ethstore/tests/util/transient_dir.rs index 150ae8108..45e2aab09 100755 --- a/ethstore/tests/util/transient_dir.rs +++ b/ethstore/tests/util/transient_dir.rs @@ -74,4 +74,8 @@ impl KeyDirectory for TransientDir { fn remove(&self, account: &SafeAccount) -> Result<(), Error> { self.dir.remove(account) } + + fn unique_repr(&self) -> Result { + self.dir.unique_repr() + } }