From 1b4f67151f91d871850420ecf2c149cf7db3460a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 27 Jun 2016 09:16:34 +0200 Subject: [PATCH 01/45] fatdb and fatdb iterator module --- util/src/hashdb.rs | 10 +++ util/src/journaldb/archivedb.rs | 26 +++++++ util/src/memorydb.rs | 17 +++++ util/src/trie/fatdb.rs | 122 ++++++++++++++++++++++++++++++++ util/src/trie/mod.rs | 7 +- util/src/trie/sectriedb.rs | 6 +- util/src/trie/sectriedbmut.rs | 10 +-- util/src/trie/triedb.rs | 2 +- util/src/trie/triedbmut.rs | 4 +- 9 files changed, 191 insertions(+), 13 deletions(-) create mode 100644 util/src/trie/fatdb.rs diff --git a/util/src/hashdb.rs b/util/src/hashdb.rs index 1ec3069d8..35606e1ba 100644 --- a/util/src/hashdb.rs +++ b/util/src/hashdb.rs @@ -104,6 +104,16 @@ pub trait HashDB: AsHashDB { /// } /// ``` fn remove(&mut self, key: &H256); + + /// Insert auxiliary data into hashdb. + fn insert_aux(&mut self, _hash: Vec, _value: Vec) { + unimplemented!(); + } + + /// Get auxiliary data from hashdb. + fn get_aux(&self, _hash: &[u8]) -> Option> { + unimplemented!(); + } } /// Upcast trait. diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index c7f6b92fa..40b322802 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -26,6 +26,8 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; +const AUX_FLAG: u8 = 255; + /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -126,6 +128,23 @@ impl HashDB for ArchiveDB { fn remove(&mut self, key: &H256) { self.overlay.remove(key); } + + fn insert_aux(&mut self, hash: Vec, value: Vec) { + self.overlay.insert_aux(hash, value); + } + + fn get_aux(&self, hash: &[u8]) -> Option> { + if let Some(res) = self.overlay.get_aux(hash) { + return Some(res) + } + + let mut db_hash = hash.to_vec(); + db_hash.push(AUX_FLAG); + + self.backing.get(&db_hash) + .expect("Low-level database error. Some issue with your hard disk?") + .map(|v| v.to_vec()) + } } impl JournalDB for ArchiveDB { @@ -149,6 +168,7 @@ impl JournalDB for ArchiveDB { let batch = DBTransaction::new(); let mut inserts = 0usize; let mut deletes = 0usize; + for i in self.overlay.drain().into_iter() { let (key, (value, rc)) = i; if rc > 0 { @@ -161,6 +181,12 @@ impl JournalDB for ArchiveDB { deletes += 1; } } + + for (mut key, value) in self.overlay.drain_aux().into_iter() { + key.push(AUX_FLAG); + batch.put(&key, &value).expect("Low-level database error. Some issue with your hard disk?"); + } + if self.latest_era.map_or(true, |e| now > e) { try!(batch.put(&LATEST_ERA_KEY, &encode(&now))); self.latest_era = Some(now); diff --git a/util/src/memorydb.rs b/util/src/memorydb.rs index ea77f4aa2..f62c6b9b2 100644 --- a/util/src/memorydb.rs +++ b/util/src/memorydb.rs @@ -74,6 +74,7 @@ use std::default::Default; pub struct MemoryDB { data: HashMap, static_null_rlp: (Bytes, i32), + aux: HashMap, } impl Default for MemoryDB { @@ -88,6 +89,7 @@ impl MemoryDB { MemoryDB { data: HashMap::new(), static_null_rlp: (vec![0x80u8; 1], 1), + aux: HashMap::new(), } } @@ -139,6 +141,13 @@ impl MemoryDB { data } + /// Return the internal map of auxiliary data, clearing the current state. + pub fn drain_aux(&mut self) -> HashMap { + let mut aux = HashMap::new(); + mem::swap(&mut self.aux, &mut aux); + aux + } + /// Denote than an existing value has the given key. Used when a key gets removed without /// a prior insert and thus has a negative reference with no value. /// @@ -233,6 +242,14 @@ impl HashDB for MemoryDB { self.data.insert(key.clone(), (Bytes::new(), -1)); } } + + fn insert_aux(&mut self, hash: Vec, value: Vec) { + self.aux.insert(hash, value); + } + + fn get_aux(&self, hash: &[u8]) -> Option> { + self.aux.get(hash).cloned() + } } #[test] diff --git a/util/src/trie/fatdb.rs b/util/src/trie/fatdb.rs new file mode 100644 index 000000000..e9308d0fe --- /dev/null +++ b/util/src/trie/fatdb.rs @@ -0,0 +1,122 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use hash::H256; +use sha3::Hashable; +use hashdb::HashDB; +use super::{TrieDBMut, Trie, TrieDB, TrieMut, TrieDBIterator, TrieError}; + +/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// +/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. +pub struct FatDB<'db> { + raw: TrieDBMut<'db>, +} + +impl<'db> FatDB<'db> { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + FatDB { raw: TrieDBMut::new(db, root) } + } + + /// Create a new trie with the backing database `db` and `root`. + /// + /// Returns an error if root does not exist. + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result { + Ok(FatDB { raw: try!(TrieDBMut::from_existing(db, root)) }) + } + + /// Get the backing database. + pub fn db(&self) -> &HashDB { + self.raw.db() + } + + /// Get the backing database. + pub fn db_mut(&mut self) -> &mut HashDB { + self.raw.db_mut() + } +} + +impl<'db> Trie for FatDB<'db> { + fn root(&self) -> &H256 { + self.raw.root() + } + + fn contains(&self, key: &[u8]) -> bool { + self.raw.contains(&key.sha3()) + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.raw.get(&key.sha3()) + } +} + +impl<'db> TrieMut for FatDB<'db> { + fn insert(&mut self, key: &[u8], value: &[u8]) { + let hash = key.sha3(); + self.raw.insert(&hash, value); + let db = self.raw.db_mut(); + db.insert_aux(hash.to_vec(), key.to_vec()); + } + + fn remove(&mut self, key: &[u8]) { + self.raw.remove(&key.sha3()); + } +} + +/// Itarator over inserted pairs of key values. +pub struct FatDBIterator<'db> { + trie_iterator: TrieDBIterator<'db>, + trie: &'db TrieDB<'db>, +} + +impl<'db> FatDBIterator<'db> { + pub fn new(trie: &'db TrieDB) -> Self { + FatDBIterator { + trie_iterator: TrieDBIterator::new(trie), + trie: trie, + } + } +} + +impl<'db> Iterator for FatDBIterator<'db> { + type Item = (Vec, &'db [u8]); + + fn next(&mut self) -> Option { + self.trie_iterator.next() + .map(|(hash, value)| { + (self.trie.db().get_aux(&hash).expect("Missing fatdb hash"), value) + }) + } +} + +#[test] +fn fatdb_to_trie() { + use memorydb::MemoryDB; + use super::TrieDB; + + let mut memdb = MemoryDB::new(); + let mut root = H256::default(); + { + let mut t = FatDB::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } + let t = TrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]); + assert_eq!(FatDBIterator::new(&t).collect::>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]); +} diff --git a/util/src/trie/mod.rs b/util/src/trie/mod.rs index 79135162c..78b91b04a 100644 --- a/util/src/trie/mod.rs +++ b/util/src/trie/mod.rs @@ -35,12 +35,15 @@ pub mod sectriedb; /// Export the sectriedbmut module. pub mod sectriedbmut; +mod fatdb; + pub use self::trietraits::{Trie, TrieMut}; pub use self::standardmap::{Alphabet, StandardMap, ValueMode}; pub use self::triedbmut::TrieDBMut; -pub use self::triedb::TrieDB; +pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; +pub use self::fatdb::{FatDB, FatDBIterator}; /// Trie Errors #[derive(Debug)] @@ -53,4 +56,4 @@ impl fmt::Display for TrieError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Trie Error: Invalid state root.") } -} \ No newline at end of file +} diff --git a/util/src/trie/sectriedb.rs b/util/src/trie/sectriedb.rs index eda4ac58b..f7f0b62cb 100644 --- a/util/src/trie/sectriedb.rs +++ b/util/src/trie/sectriedb.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use hash::*; -use sha3::*; +use hash::H256; +use sha3::Hashable; use hashdb::HashDB; use super::triedb::TrieDB; use super::trietraits::Trie; @@ -68,7 +68,7 @@ fn trie_to_sectrie() { use super::trietraits::TrieMut; let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut root = H256::default(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]); diff --git a/util/src/trie/sectriedbmut.rs b/util/src/trie/sectriedbmut.rs index 9c8231af4..43e17a1b0 100644 --- a/util/src/trie/sectriedbmut.rs +++ b/util/src/trie/sectriedbmut.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use hash::*; -use sha3::*; +use hash::H256; +use sha3::Hashable; use hashdb::HashDB; use super::triedbmut::TrieDBMut; use super::trietraits::{Trie, TrieMut}; @@ -44,10 +44,10 @@ impl<'db> SecTrieDBMut<'db> { } /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { self.raw.db() } + pub fn db(&self) -> &HashDB { self.raw.db() } /// Get the backing database. - pub fn db_mut(&'db mut self) -> &'db mut HashDB { self.raw.db_mut() } + pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } impl<'db> Trie for SecTrieDBMut<'db> { @@ -78,7 +78,7 @@ fn sectrie_to_trie() { use super::triedb::*; let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut root = H256::default(); { let mut t = SecTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 9cd3a8c41..b3d20d56f 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -257,7 +257,7 @@ pub struct TrieDBIterator<'a> { impl<'a> TrieDBIterator<'a> { /// Create a new iterator. - fn new(db: &'a TrieDB) -> TrieDBIterator<'a> { + pub fn new(db: &'a TrieDB) -> TrieDBIterator<'a> { let mut r = TrieDBIterator { db: db, trail: vec![], diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index 65d5bbd2b..ed6ba7634 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -99,12 +99,12 @@ impl<'db> TrieDBMut<'db> { } /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { + pub fn db(&self) -> &HashDB { self.db } /// Get the backing database. - pub fn db_mut(&'db mut self) -> &'db mut HashDB { + pub fn db_mut(&mut self) -> &mut HashDB { self.db } From 5ecbeaa82f31e51994fdb870e19ff7c7595dff03 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 27 Jun 2016 10:59:59 +0200 Subject: [PATCH 02/45] trie factory in progress --- util/src/trie/fatdb.rs | 53 ++++++++------------- util/src/trie/fatdbmut.rs | 95 +++++++++++++++++++++++++++++++++++++ util/src/trie/mod.rs | 59 +++++++++++++++++++++++ util/src/trie/sectriedb.rs | 4 ++ util/src/trie/triedb.rs | 10 +++- util/src/trie/trietraits.rs | 3 ++ 6 files changed, 190 insertions(+), 34 deletions(-) create mode 100644 util/src/trie/fatdbmut.rs diff --git a/util/src/trie/fatdb.rs b/util/src/trie/fatdb.rs index e9308d0fe..1b520d8ad 100644 --- a/util/src/trie/fatdb.rs +++ b/util/src/trie/fatdb.rs @@ -17,28 +17,25 @@ use hash::H256; use sha3::Hashable; use hashdb::HashDB; -use super::{TrieDBMut, Trie, TrieDB, TrieMut, TrieDBIterator, TrieError}; +use super::{TrieDB, Trie, TrieDBIterator, TrieError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// -/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. +/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDB` object. pub struct FatDB<'db> { - raw: TrieDBMut<'db>, + raw: TrieDB<'db>, } impl<'db> FatDB<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { - FatDB { raw: TrieDBMut::new(db, root) } - } + pub fn new(db: &'db HashDB, root: &'db H256) -> Result { + let fatdb = FatDB { + raw: try!(TrieDB::new(db, root)) + }; - /// Create a new trie with the backing database `db` and `root`. - /// - /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result { - Ok(FatDB { raw: try!(TrieDBMut::from_existing(db, root)) }) + Ok(fatdb) } /// Get the backing database. @@ -46,13 +43,17 @@ impl<'db> FatDB<'db> { self.raw.db() } - /// Get the backing database. - pub fn db_mut(&mut self) -> &mut HashDB { - self.raw.db_mut() + /// Iterator over all key / vlaues in the trie. + pub fn iter(&self) -> FatDBIterator { + FatDBIterator::new(&self.raw) } } impl<'db> Trie for FatDB<'db> { + fn iter<'a>(&'a self) -> Box, &[u8])> + 'a> { + Box::new(FatDB::iter(self)) + } + fn root(&self) -> &H256 { self.raw.root() } @@ -66,19 +67,6 @@ impl<'db> Trie for FatDB<'db> { } } -impl<'db> TrieMut for FatDB<'db> { - fn insert(&mut self, key: &[u8], value: &[u8]) { - let hash = key.sha3(); - self.raw.insert(&hash, value); - let db = self.raw.db_mut(); - db.insert_aux(hash.to_vec(), key.to_vec()); - } - - fn remove(&mut self, key: &[u8]) { - self.raw.remove(&key.sha3()); - } -} - /// Itarator over inserted pairs of key values. pub struct FatDBIterator<'db> { trie_iterator: TrieDBIterator<'db>, @@ -86,6 +74,7 @@ pub struct FatDBIterator<'db> { } impl<'db> FatDBIterator<'db> { + /// Creates new iterator. pub fn new(trie: &'db TrieDB) -> Self { FatDBIterator { trie_iterator: TrieDBIterator::new(trie), @@ -108,15 +97,15 @@ impl<'db> Iterator for FatDBIterator<'db> { #[test] fn fatdb_to_trie() { use memorydb::MemoryDB; - use super::TrieDB; + use trie::{FatDBMut, TrieMut}; let mut memdb = MemoryDB::new(); let mut root = H256::default(); { - let mut t = FatDB::new(&mut memdb, &mut root); + let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); } - let t = TrieDB::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]); - assert_eq!(FatDBIterator::new(&t).collect::>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]); + let t = FatDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), &[0x01u8, 0x23]); + assert_eq!(t.iter().collect::>(), vec![(vec![0x01u8, 0x23], &[0x01u8, 0x23] as &[u8])]); } diff --git a/util/src/trie/fatdbmut.rs b/util/src/trie/fatdbmut.rs new file mode 100644 index 000000000..b04b3dcaa --- /dev/null +++ b/util/src/trie/fatdbmut.rs @@ -0,0 +1,95 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use hash::H256; +use sha3::Hashable; +use hashdb::HashDB; +use super::{TrieDBMut, Trie, TrieMut, TrieError}; + +/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// +/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. +pub struct FatDBMut<'db> { + raw: TrieDBMut<'db>, +} + +impl<'db> FatDBMut<'db> { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + FatDBMut { raw: TrieDBMut::new(db, root) } + } + + /// Create a new trie with the backing database `db` and `root`. + /// + /// Returns an error if root does not exist. + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Result { + Ok(FatDBMut { raw: try!(TrieDBMut::from_existing(db, root)) }) + } + + /// Get the backing database. + pub fn db(&self) -> &HashDB { + self.raw.db() + } + + /// Get the backing database. + pub fn db_mut(&mut self) -> &mut HashDB { + self.raw.db_mut() + } +} + +impl<'db> Trie for FatDBMut<'db> { + fn root(&self) -> &H256 { + self.raw.root() + } + + fn contains(&self, key: &[u8]) -> bool { + self.raw.contains(&key.sha3()) + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.raw.get(&key.sha3()) + } +} + +impl<'db> TrieMut for FatDBMut<'db> { + fn insert(&mut self, key: &[u8], value: &[u8]) { + let hash = key.sha3(); + self.raw.insert(&hash, value); + let db = self.raw.db_mut(); + db.insert_aux(hash.to_vec(), key.to_vec()); + } + + fn remove(&mut self, key: &[u8]) { + self.raw.remove(&key.sha3()); + } +} + +#[test] +fn fatdb_to_trie() { + use memorydb::MemoryDB; + use super::TrieDB; + + let mut memdb = MemoryDB::new(); + let mut root = H256::default(); + { + let mut t = FatDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } + let t = TrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]); +} diff --git a/util/src/trie/mod.rs b/util/src/trie/mod.rs index 78b91b04a..81d7fd774 100644 --- a/util/src/trie/mod.rs +++ b/util/src/trie/mod.rs @@ -17,6 +17,8 @@ //! Trie interface and implementation. use std::fmt; +use hash::H256; +use hashdb::HashDB; /// Export the trietraits module. pub mod trietraits; @@ -37,6 +39,8 @@ pub mod sectriedbmut; mod fatdb; +mod fatdbmut; + pub use self::trietraits::{Trie, TrieMut}; pub use self::standardmap::{Alphabet, StandardMap, ValueMode}; pub use self::triedbmut::TrieDBMut; @@ -44,6 +48,7 @@ pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; +pub use self::fatdbmut::FatDBMut; /// Trie Errors #[derive(Debug)] @@ -57,3 +62,57 @@ impl fmt::Display for TrieError { write!(f, "Trie Error: Invalid state root.") } } + +/// Trie types +#[derive(Debug, Clone)] +pub enum TrieSpec { + /// Secure trie. + Secure, + /// Secure trie with fat database. + Fat, +} + +impl Default for TrieSpec { + fn default() -> TrieSpec { + TrieSpec::Secure + } +} + +/// Trie factory. +#[derive(Default, Clone)] +pub struct TrieFactory { + spec: TrieSpec, +} + +impl TrieFactory { + /// Creates new factory. + pub fn new(spec: TrieSpec) -> Self { + TrieFactory { + spec: spec, + } + } + + /// Create new immutable instance of Trie. + pub fn create<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result, TrieError> { + match self.spec { + TrieSpec::Secure => Ok(Box::new(try!(SecTrieDB::new(db, root)))), + TrieSpec::Fat => Ok(Box::new(try!(FatDB::new(db, root)))), + } + } + + /// Create new mutable instance of Trie. + pub fn create_mut<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result, TrieError> { + match self.spec { + TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::new(db, root))), + TrieSpec::Fat => Ok(Box::new(FatDBMut::new(db, root))), + } + } + + /// Create new mutable instance of trie and check for errors. + pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result, TrieError> { + match self.spec { + TrieSpec::Secure => Ok(Box::new(try!(SecTrieDBMut::from_existing(db, root)))), + TrieSpec::Fat => Ok(Box::new(try!(FatDBMut::from_existing(db, root)))), + } + } +} diff --git a/util/src/trie/sectriedb.rs b/util/src/trie/sectriedb.rs index f7f0b62cb..99770e382 100644 --- a/util/src/trie/sectriedb.rs +++ b/util/src/trie/sectriedb.rs @@ -50,6 +50,10 @@ impl<'db> SecTrieDB<'db> { } impl<'db> Trie for SecTrieDB<'db> { + fn iter<'a>(&'a self) -> Box, &[u8])> + 'a> { + Box::new(TrieDB::iter(&self.raw)) + } + fn root(&self) -> &H256 { self.raw.root() } fn contains(&self, key: &[u8]) -> bool { diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index b3d20d56f..c88ec422e 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -18,7 +18,7 @@ use common::*; use hashdb::*; use nibbleslice::*; use rlp::*; -use super::trietraits::Trie; +use super::trietraits::{Trie}; use super::node::Node; use super::TrieError; @@ -331,10 +331,16 @@ impl<'a> Iterator for TrieDBIterator<'a> { impl<'db> TrieDB<'db> { /// Get all keys/values stored in the trie. - pub fn iter(&self) -> TrieDBIterator { TrieDBIterator::new(self) } + pub fn iter(&self) -> TrieDBIterator { + TrieDBIterator::new(self) + } } impl<'db> Trie for TrieDB<'db> { + fn iter<'a>(&'a self) -> Box, &[u8])> + 'a> { + Box::new(TrieDB::iter(self)) + } + fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { diff --git a/util/src/trie/trietraits.rs b/util/src/trie/trietraits.rs index 160f9466f..03f5e6788 100644 --- a/util/src/trie/trietraits.rs +++ b/util/src/trie/trietraits.rs @@ -19,6 +19,9 @@ use rlp::SHA3_NULL_RLP; /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { + /// Returns an iterator over elements of trie. + fn iter<'a>(&'a self) -> Box, &[u8])> + 'a> { unimplemented!() } + /// Return the root of the trie. fn root(&self) -> &H256; From 36626f96a81ebd43c815a4e865096265fd812b14 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 27 Jun 2016 11:19:27 +0200 Subject: [PATCH 03/45] separated TrieMut from Trie, added Generic Trie type to TrieFactory --- util/src/trie/fatdbmut.rs | 4 +--- util/src/trie/mod.rs | 13 +++++++++---- util/src/trie/sectriedbmut.rs | 4 +--- util/src/trie/triedbmut.rs | 4 +--- util/src/trie/trietraits.rs | 21 ++++++++++++++++----- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/util/src/trie/fatdbmut.rs b/util/src/trie/fatdbmut.rs index b04b3dcaa..b699bf208 100644 --- a/util/src/trie/fatdbmut.rs +++ b/util/src/trie/fatdbmut.rs @@ -52,7 +52,7 @@ impl<'db> FatDBMut<'db> { } } -impl<'db> Trie for FatDBMut<'db> { +impl<'db> TrieMut for FatDBMut<'db> { fn root(&self) -> &H256 { self.raw.root() } @@ -64,9 +64,7 @@ impl<'db> Trie for FatDBMut<'db> { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { self.raw.get(&key.sha3()) } -} -impl<'db> TrieMut for FatDBMut<'db> { fn insert(&mut self, key: &[u8], value: &[u8]) { let hash = key.sha3(); self.raw.insert(&hash, value); diff --git a/util/src/trie/mod.rs b/util/src/trie/mod.rs index 81d7fd774..130406948 100644 --- a/util/src/trie/mod.rs +++ b/util/src/trie/mod.rs @@ -66,6 +66,8 @@ impl fmt::Display for TrieError { /// Trie types #[derive(Debug, Clone)] pub enum TrieSpec { + /// Generic trie. + Generic, /// Secure trie. Secure, /// Secure trie with fat database. @@ -95,22 +97,25 @@ impl TrieFactory { /// Create new immutable instance of Trie. pub fn create<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result, TrieError> { match self.spec { + TrieSpec::Generic => Ok(Box::new(try!(TrieDB::new(db, root)))), TrieSpec::Secure => Ok(Box::new(try!(SecTrieDB::new(db, root)))), TrieSpec::Fat => Ok(Box::new(try!(FatDB::new(db, root)))), } } /// Create new mutable instance of Trie. - pub fn create_mut<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result, TrieError> { + pub fn create_mut<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box { match self.spec { - TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::new(db, root))), - TrieSpec::Fat => Ok(Box::new(FatDBMut::new(db, root))), + TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), + TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), + TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), } } /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result, TrieError> { + pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result, TrieError> { match self.spec { + TrieSpec::Generic => Ok(Box::new(try!(TrieDBMut::from_existing(db, root)))), TrieSpec::Secure => Ok(Box::new(try!(SecTrieDBMut::from_existing(db, root)))), TrieSpec::Fat => Ok(Box::new(try!(FatDBMut::from_existing(db, root)))), } diff --git a/util/src/trie/sectriedbmut.rs b/util/src/trie/sectriedbmut.rs index 43e17a1b0..09ff48f7a 100644 --- a/util/src/trie/sectriedbmut.rs +++ b/util/src/trie/sectriedbmut.rs @@ -50,7 +50,7 @@ impl<'db> SecTrieDBMut<'db> { pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } -impl<'db> Trie for SecTrieDBMut<'db> { +impl<'db> TrieMut for SecTrieDBMut<'db> { fn root(&self) -> &H256 { self.raw.root() } fn contains(&self, key: &[u8]) -> bool { @@ -60,9 +60,7 @@ impl<'db> Trie for SecTrieDBMut<'db> { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { self.raw.get(&key.sha3()) } -} -impl<'db> TrieMut for SecTrieDBMut<'db> { fn insert(&mut self, key: &[u8], value: &[u8]) { self.raw.insert(&key.sha3(), value); } diff --git a/util/src/trie/triedbmut.rs b/util/src/trie/triedbmut.rs index ed6ba7634..6d34096fc 100644 --- a/util/src/trie/triedbmut.rs +++ b/util/src/trie/triedbmut.rs @@ -642,7 +642,7 @@ impl<'db> TrieDBMut<'db> { } } -impl<'db> Trie for TrieDBMut<'db> { +impl<'db> TrieMut for TrieDBMut<'db> { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { @@ -652,9 +652,7 @@ impl<'db> Trie for TrieDBMut<'db> { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { self.do_lookup(&NibbleSlice::new(key)) } -} -impl<'db> TrieMut for TrieDBMut<'db> { fn insert(&mut self, key: &[u8], value: &[u8]) { match value.is_empty() { false => self.insert_ns(&NibbleSlice::new(key), value), diff --git a/util/src/trie/trietraits.rs b/util/src/trie/trietraits.rs index 03f5e6788..fb53350c9 100644 --- a/util/src/trie/trietraits.rs +++ b/util/src/trie/trietraits.rs @@ -19,9 +19,6 @@ use rlp::SHA3_NULL_RLP; /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { - /// Returns an iterator over elements of trie. - fn iter<'a>(&'a self) -> Box, &[u8])> + 'a> { unimplemented!() } - /// Return the root of the trie. fn root(&self) -> &H256; @@ -33,10 +30,25 @@ pub trait Trie { /// What is the value of the given key in this trie? fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; + + /// Returns an iterator over elements of trie. + fn iter<'a>(&'a self) -> Box, &[u8])> + 'a>; } /// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait TrieMut: Trie { +pub trait TrieMut { + /// Return the root of the trie. + fn root(&self) -> &H256; + + /// Is the trie empty? + fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } + + /// Does the trie contain a given key? + fn contains(&self, key: &[u8]) -> bool; + + /// What is the value of the given key in this trie? + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; + /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing /// `key` from the trie. fn insert(&mut self, key: &[u8], value: &[u8]); @@ -45,4 +57,3 @@ pub trait TrieMut: Trie { /// value. fn remove(&mut self, key: &[u8]); } - From 7904464d248b205193735f74feb394d927eb15de Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 27 Jun 2016 13:37:22 +0200 Subject: [PATCH 04/45] use mem::replace instead of mem::swap in ArchiveDB, add aux_remove() --- util/src/hashdb.rs | 5 +++++ util/src/journaldb/archivedb.rs | 13 +++++++++++-- util/src/memorydb.rs | 12 ++++++------ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/util/src/hashdb.rs b/util/src/hashdb.rs index 35606e1ba..51b9d445e 100644 --- a/util/src/hashdb.rs +++ b/util/src/hashdb.rs @@ -114,6 +114,11 @@ pub trait HashDB: AsHashDB { fn get_aux(&self, _hash: &[u8]) -> Option> { unimplemented!(); } + + /// Removes auxiliary data from hashdb. + fn remove_aux(&mut self, _hash: &[u8]) { + unimplemented!(); + } } /// Upcast trait. diff --git a/util/src/journaldb/archivedb.rs b/util/src/journaldb/archivedb.rs index 40b322802..3713fc79e 100644 --- a/util/src/journaldb/archivedb.rs +++ b/util/src/journaldb/archivedb.rs @@ -26,8 +26,13 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; +/// Suffix appended to auxiliary keys to distinguish them from normal keys. +/// Would be nich to use rocksdb columns for this eventually. const AUX_FLAG: u8 = 255; +/// Database version. +const DB_VERSION : u32 = 0x103; + /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -41,8 +46,6 @@ pub struct ArchiveDB { latest_era: Option, } -const DB_VERSION : u32 = 0x103; - impl ArchiveDB { /// Create a new instance from file pub fn new(path: &str, cache_size: Option) -> ArchiveDB { @@ -122,9 +125,11 @@ impl HashDB for ArchiveDB { fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } + fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); } + fn remove(&mut self, key: &H256) { self.overlay.remove(key); } @@ -145,6 +150,10 @@ impl HashDB for ArchiveDB { .expect("Low-level database error. Some issue with your hard disk?") .map(|v| v.to_vec()) } + + fn remove_aux(&mut self, hash: &[u8]) { + self.overlay.remove_aux(hash); + } } impl JournalDB for ArchiveDB { diff --git a/util/src/memorydb.rs b/util/src/memorydb.rs index f62c6b9b2..f63dfd992 100644 --- a/util/src/memorydb.rs +++ b/util/src/memorydb.rs @@ -136,16 +136,12 @@ impl MemoryDB { /// Return the internal map of hashes to data, clearing the current state. pub fn drain(&mut self) -> HashMap { - let mut data = HashMap::new(); - mem::swap(&mut self.data, &mut data); - data + mem::replace(&mut self.data, HashMap::new()) } /// Return the internal map of auxiliary data, clearing the current state. pub fn drain_aux(&mut self) -> HashMap { - let mut aux = HashMap::new(); - mem::swap(&mut self.aux, &mut aux); - aux + mem::replace(&mut self.aux, HashMap::new()) } /// Denote than an existing value has the given key. Used when a key gets removed without @@ -250,6 +246,10 @@ impl HashDB for MemoryDB { fn get_aux(&self, hash: &[u8]) -> Option> { self.aux.get(hash).cloned() } + + fn remove_aux(&mut self, hash: &[u8]) { + self.aux.remove(hash); + } } #[test] From 06cf2a3f418c4c3cdc36d2b661df7666083524ff Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 27 Jun 2016 13:50:08 +0200 Subject: [PATCH 05/45] updated FatDB description --- util/src/trie/fatdb.rs | 5 +++-- util/src/trie/fatdbmut.rs | 3 ++- util/src/trie/mod.rs | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/util/src/trie/fatdb.rs b/util/src/trie/fatdb.rs index 1b520d8ad..555e4644f 100644 --- a/util/src/trie/fatdb.rs +++ b/util/src/trie/fatdb.rs @@ -19,9 +19,10 @@ use sha3::Hashable; use hashdb::HashDB; use super::{TrieDB, Trie, TrieDBIterator, TrieError}; -/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// Additionaly it stores inserted hash-key mappings for later retrieval. /// -/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDB` object. +/// Use it as a `Trie` or `TrieMut` trait object. pub struct FatDB<'db> { raw: TrieDB<'db>, } diff --git a/util/src/trie/fatdbmut.rs b/util/src/trie/fatdbmut.rs index b699bf208..3e72c0653 100644 --- a/util/src/trie/fatdbmut.rs +++ b/util/src/trie/fatdbmut.rs @@ -20,8 +20,9 @@ use hashdb::HashDB; use super::{TrieDBMut, Trie, TrieMut, TrieError}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// Additionaly it stores inserted hash-key mappings for later retrieval. /// -/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. +/// Use it as a `Trie` or `TrieMut` trait object. pub struct FatDBMut<'db> { raw: TrieDBMut<'db>, } diff --git a/util/src/trie/mod.rs b/util/src/trie/mod.rs index 130406948..f6e731370 100644 --- a/util/src/trie/mod.rs +++ b/util/src/trie/mod.rs @@ -95,7 +95,7 @@ impl TrieFactory { } /// Create new immutable instance of Trie. - pub fn create<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result, TrieError> { + pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result, TrieError> { match self.spec { TrieSpec::Generic => Ok(Box::new(try!(TrieDB::new(db, root)))), TrieSpec::Secure => Ok(Box::new(try!(SecTrieDB::new(db, root)))), @@ -104,7 +104,7 @@ impl TrieFactory { } /// Create new mutable instance of Trie. - pub fn create_mut<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box { + pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box { match self.spec { TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), From c096c087dfc12e9c37fcf68016b6b999fc470a81 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 15:23:41 +0200 Subject: [PATCH 06/45] Ensure we don't reject our own transactions for gasprice. (#1485) * Ensure we don't reject our own transactions for gasprice. * Add test. --- ethcore/src/miner/transaction_queue.rs | 21 +++++++++++++++++++-- rpc/src/v1/impls/mod.rs | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ethcore/src/miner/transaction_queue.rs b/ethcore/src/miner/transaction_queue.rs index 17ca18272..7f5b59c38 100644 --- a/ethcore/src/miner/transaction_queue.rs +++ b/ethcore/src/miner/transaction_queue.rs @@ -441,7 +441,7 @@ impl TransactionQueue { trace!(target: "miner", "Importing: {:?}", tx.hash()); - if tx.gas_price < self.minimal_gas_price { + if tx.gas_price < self.minimal_gas_price && origin != TransactionOrigin::Local { trace!(target: "miner", "Dropping transaction below minimal gas price threshold: {:?} (gp: {} < {})", tx.hash(), @@ -1055,7 +1055,7 @@ mod test { } #[test] - fn should_not_import_transaction_below_min_gas_price_threshold() { + fn should_not_import_transaction_below_min_gas_price_threshold_if_external() { // given let mut txq = TransactionQueue::new(); let tx = new_tx(); @@ -1074,6 +1074,23 @@ mod test { assert_eq!(stats.future, 0); } + #[test] + fn should_import_transaction_below_min_gas_price_threshold_if_local() { + // given + let mut txq = TransactionQueue::new(); + let tx = new_tx(); + txq.set_minimal_gas_price(tx.gas_price + U256::one()); + + // when + let res = txq.add(tx, &default_nonce, TransactionOrigin::Local); + + // then + assert_eq!(res.unwrap(), TransactionImportResult::Current); + let stats = txq.status(); + assert_eq!(stats.pending, 1); + assert_eq!(stats.future, 0); + } + #[test] fn should_reject_incorectly_signed_transaction() { // given diff --git a/rpc/src/v1/impls/mod.rs b/rpc/src/v1/impls/mod.rs index fdd2de8fa..1a6b1c398 100644 --- a/rpc/src/v1/impls/mod.rs +++ b/rpc/src/v1/impls/mod.rs @@ -165,7 +165,7 @@ fn transaction_error(error: EthcoreError) -> Error { "There is too many transactions in the queue. Your transaction was dropped due to limit. Try increasing the fee.".into() }, InsufficientGasPrice { minimal, got } => { - format!("Transaction fee is to low. It does not satisfy your node's minimal fee (minimal: {}, got: {}). Try increasing the fee.", minimal, got) + format!("Transaction fee is too low. It does not satisfy your node's minimal fee (minimal: {}, got: {}). Try increasing the fee.", minimal, got) }, InsufficientBalance { balance, cost } => { format!("Insufficient funds. Account you try to send transaction from does not have enough funds. Required {} and got: {}.", cost, balance) From 7c27f9dfed2d9e2da645b3f02293d1223c75b008 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 15:37:11 +0200 Subject: [PATCH 07/45] Add CLI option and route to MinerOptions. --- ethcore/src/miner/miner.rs | 3 +++ parity/cli.rs | 3 +++ parity/configuration.rs | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 803706c56..8743edff3 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -44,6 +44,8 @@ pub enum PendingSet { /// Configures the behaviour of the miner. #[derive(Debug)] pub struct MinerOptions { + /// URLs to notify when there is new work. + pub new_work_notify: Vec, /// Force the miner to reseal, even when nobody has asked for work. pub force_sealing: bool, /// Reseal on receipt of new external transactions. @@ -61,6 +63,7 @@ pub struct MinerOptions { impl Default for MinerOptions { fn default() -> Self { MinerOptions { + new_work_notify: vec![], force_sealing: false, reseal_on_external_tx: true, reseal_on_own_tx: true, diff --git a/parity/cli.rs b/parity/cli.rs index 27578df73..a8e53eeff 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -162,6 +162,8 @@ Sealing/Mining Options: more than 32 characters. --tx-queue-size LIMIT Maximum amount of transactions in the queue (waiting to be included in next block) [default: 1024]. + --notify-work URLS URLs to which work package notifications are pushed. + URLS should be a comma-delimited list of HTTP URLs. Footprint Options: --tracing BOOL Indicates if full transaction tracing should be @@ -311,6 +313,7 @@ pub struct Args { pub flag_gas_cap: String, pub flag_extra_data: Option, pub flag_tx_queue_size: usize, + pub flag_work_notify: Option, pub flag_logging: Option, pub flag_version: bool, pub flag_from: String, diff --git a/parity/configuration.rs b/parity/configuration.rs index 121d40e6e..aebea8e61 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -83,6 +83,10 @@ impl Configuration { ) } + pub fn work_notify(&self) -> Vec { + self.args.flag_work_notify.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) + } + pub fn miner_options(&self) -> MinerOptions { let (own, ext) = match self.args.flag_reseal_on_txs.as_str() { "none" => (false, false), @@ -92,6 +96,7 @@ impl Configuration { x => die!("{}: Invalid value for --reseal option. Use --help for more information.", x) }; MinerOptions { + new_work_notify: self.work_notify(), force_sealing: self.args.flag_force_sealing, reseal_on_external_tx: ext, reseal_on_own_tx: own, From 05927eba1fb08b4d69cbd724181d4e32f51c7804 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 15:43:34 +0200 Subject: [PATCH 08/45] Include number in eth_getWork. --- rpc/src/v1/impls/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 05dc89564..8bfc661e3 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -498,7 +498,7 @@ impl Eth for EthClient where let pow_hash = b.hash(); let target = Ethash::difficulty_to_boundary(b.block().header().difficulty()); let seed_hash = &self.seed_compute.lock().unwrap().get_seedhash(b.block().header().number()); - to_value(&(pow_hash, H256::from_slice(&seed_hash[..]), target)) + to_value(&(pow_hash, H256::from_slice(&seed_hash[..]), target, &U256::from(b.block().header().number()))) }).unwrap_or(Err(Error::internal_error())) // no work found. }, _ => Err(Error::invalid_params()) From 580913fa7dbe1291be493451b80f7e449e52f17f Mon Sep 17 00:00:00 2001 From: NikVolf Date: Wed, 29 Jun 2016 17:23:29 +0300 Subject: [PATCH 09/45] vm factory to mining client --- ethcore/src/client/client.rs | 7 ++++--- ethcore/src/client/mod.rs | 6 +++--- ethcore/src/client/test_client.rs | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 1a437bfac..5d157b654 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -515,9 +515,6 @@ impl BlockChainClient for Client { ret } - fn vm_factory(&self) -> &EvmFactory { - &self.vm_factory - } fn block_header(&self, id: BlockID) -> Option { Self::block_hash(&self.chain, id).and_then(|hash| self.chain.block(&hash).map(|bytes| BlockView::new(&bytes).rlp().at(0).as_raw().to_vec())) @@ -829,6 +826,10 @@ impl MiningBlockChainClient for Client { open_block } + + fn vm_factory(&self) -> &EvmFactory { + &self.vm_factory + } } impl MayPanic for Client { diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index 2098d8a2f..bef814b4e 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -172,9 +172,6 @@ pub trait BlockChainClient : Sync + Send { // TODO: should be able to accept blockchain location for call. fn call(&self, t: &SignedTransaction, analytics: CallAnalytics) -> Result; - /// Returns EvmFactory. - fn vm_factory(&self) -> &EvmFactory; - /// Returns traces matching given filter. fn filter_traces(&self, filter: TraceFilter) -> Option>; @@ -253,4 +250,7 @@ pub trait MiningBlockChainClient : BlockChainClient { /// Returns OpenBlock prepared for closing. fn prepare_open_block(&self, author: Address, gas_range_target: (U256, U256), extra_data: Bytes) -> OpenBlock; + + /// Returns EvmFactory. + fn vm_factory(&self) -> &EvmFactory; } diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index a7f508a51..ed1f10e09 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -244,6 +244,10 @@ impl MiningBlockChainClient for TestBlockChainClient { fn prepare_open_block(&self, _author: Address, _gas_range_target: (U256, U256), _extra_data: Bytes) -> OpenBlock { unimplemented!(); } + + fn vm_factory(&self) -> &EvmFactory { + unimplemented!(); + } } impl BlockChainClient for TestBlockChainClient { @@ -463,10 +467,6 @@ impl BlockChainClient for TestBlockChainClient { } } - fn vm_factory(&self) -> &EvmFactory { - unimplemented!(); - } - fn filter_traces(&self, _filter: TraceFilter) -> Option> { unimplemented!(); } From 5d1ff3d7ba518a12f72541b96559c62d9b0e8571 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 16:26:19 +0200 Subject: [PATCH 10/45] Introduce options for fine-grained management of work queue. (#1484) * Introduce options for fine-grained management of work queue. - Minimum reseal period between non-mandatory (transaction) reseals. - Maximum historical cached block size. Defaults changed to reflect real-world scenarios (2s, 20 blocks). * Fix test bug. * 50 -> 20. --- ethcore/src/miner/miner.rs | 24 +++++++++++++++++++----- parity/cli.rs | 9 +++++++++ parity/configuration.rs | 3 +++ rpc/src/v1/tests/eth.rs | 3 +++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 803706c56..4518e416a 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -16,6 +16,7 @@ use rayon::prelude::*; use std::sync::atomic::AtomicBool; +use std::time::{Instant, Duration}; use util::*; use account_provider::AccountProvider; @@ -50,12 +51,16 @@ pub struct MinerOptions { pub reseal_on_external_tx: bool, /// Reseal on receipt of new local transactions. pub reseal_on_own_tx: bool, + /// Minimum period between transaction-inspired reseals. + pub reseal_min_period: Duration, /// Maximum amount of gas to bother considering for block insertion. pub tx_gas_limit: U256, /// Maximum size of the transaction queue. pub tx_queue_size: usize, /// Whether we should fallback to providing all the queue's transactions or just pending. pub pending_set: PendingSet, + /// How many historical work packages can we store before running out? + pub work_queue_size: usize, } impl Default for MinerOptions { @@ -67,6 +72,8 @@ impl Default for MinerOptions { tx_gas_limit: !U256::zero(), tx_queue_size: 1024, pending_set: PendingSet::AlwaysQueue, + reseal_min_period: Duration::from_secs(0), + work_queue_size: 20, } } } @@ -80,6 +87,7 @@ pub struct Miner { // for sealing... options: MinerOptions, sealing_enabled: AtomicBool, + next_allowed_reseal: Mutex, sealing_block_last_request: Mutex, gas_range_target: RwLock<(U256, U256)>, author: RwLock
, @@ -96,8 +104,9 @@ impl Miner { transaction_queue: Mutex::new(TransactionQueue::new()), options: Default::default(), sealing_enabled: AtomicBool::new(false), + next_allowed_reseal: Mutex::new(Instant::now()), sealing_block_last_request: Mutex::new(0), - sealing_work: Mutex::new(UsingQueue::new(5)), + sealing_work: Mutex::new(UsingQueue::new(20)), gas_range_target: RwLock::new((U256::zero(), U256::zero())), author: RwLock::new(Address::default()), extra_data: RwLock::new(Vec::new()), @@ -111,13 +120,14 @@ impl Miner { Arc::new(Miner { transaction_queue: Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, options.tx_gas_limit)), sealing_enabled: AtomicBool::new(options.force_sealing), - options: options, + next_allowed_reseal: Mutex::new(Instant::now()), sealing_block_last_request: Mutex::new(0), - sealing_work: Mutex::new(UsingQueue::new(5)), + sealing_work: Mutex::new(UsingQueue::new(options.work_queue_size)), gas_range_target: RwLock::new((U256::zero(), U256::zero())), author: RwLock::new(Address::default()), extra_data: RwLock::new(Vec::new()), accounts: accounts, + options: options, spec: spec, }) } @@ -261,6 +271,9 @@ impl Miner { // Return if !have_work } + + /// Are we allowed to do a non-mandatory reseal? + fn tx_reseal_allowed(&self) -> bool { Instant::now() > *self.next_allowed_reseal.lock().unwrap() } } const SEALING_TIMEOUT_IN_BLOCKS : u64 = 5; @@ -431,7 +444,7 @@ impl MinerService for Miner { .map(|tx| transaction_queue.add(tx, &fetch_account, TransactionOrigin::External)) .collect() }; - if !results.is_empty() && self.options.reseal_on_external_tx { + if !results.is_empty() && self.options.reseal_on_external_tx && self.tx_reseal_allowed() { self.update_sealing(chain); } results @@ -466,7 +479,7 @@ impl MinerService for Miner { import }; - if imported.is_ok() && self.options.reseal_on_own_tx { + if imported.is_ok() && self.options.reseal_on_own_tx && self.tx_reseal_allowed() { // Make sure to do it after transaction is imported and lock is droped. // We need to create pending block and enable sealing let prepared = self.enable_and_prepare_sealing(chain); @@ -559,6 +572,7 @@ impl MinerService for Miner { self.sealing_enabled.store(false, atomic::Ordering::Relaxed); self.sealing_work.lock().unwrap().reset(); } else { + *self.next_allowed_reseal.lock().unwrap() = Instant::now() + self.options.reseal_min_period; self.prepare_sealing(chain); } } diff --git a/parity/cli.rs b/parity/cli.rs index 27578df73..a48ce3aa7 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -137,6 +137,13 @@ Sealing/Mining Options: own - reseal only on a new local transaction; ext - reseal only on a new external transaction; all - reseal on all new transactions [default: all]. + --reseal-min-period MS Specify the minimum time between reseals from + incoming transactions. MS is time measured in + milliseconds [default: 2000]. + --work-queue-size ITEMS Specify the number of historical work packages + which are kept cached lest a solution is found for + them later. High values take more memory but result + in fewer unusable solutions [default: 20]. --tx-gas-limit GAS Apply a limit of GAS as the maximum amount of gas a single transaction may have for it to be mined. --relay-set SET Set of transactions to relay. SET may be: @@ -302,6 +309,8 @@ pub struct Args { pub flag_no_token: bool, pub flag_force_sealing: bool, pub flag_reseal_on_txs: String, + pub flag_reseal_min_period: u64, + pub flag_work_queue_size: usize, pub flag_tx_gas_limit: Option, pub flag_relay_set: String, pub flag_author: Option, diff --git a/parity/configuration.rs b/parity/configuration.rs index 121d40e6e..e95ef4233 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -16,6 +16,7 @@ use std::env; use std::fs::File; +use std::time::Duration; use std::io::{BufRead, BufReader}; use std::net::{SocketAddr, IpAddr}; use std::path::PathBuf; @@ -103,6 +104,8 @@ impl Configuration { "lenient" => PendingSet::SealingOrElseQueue, x => die!("{}: Invalid value for --relay-set option. Use --help for more information.", x) }, + reseal_min_period: Duration::from_millis(self.args.flag_reseal_min_period), + work_queue_size: self.args.flag_work_queue_size, } } diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 88a12a74f..59d06e84a 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -17,6 +17,7 @@ //! rpc integration tests. use std::sync::Arc; use std::str::FromStr; +use std::time::Duration; use ethcore::client::{BlockChainClient, Client, ClientConfig}; use ethcore::ids::BlockID; @@ -57,6 +58,8 @@ fn miner_service(spec: Spec, accounts: Arc) -> Arc { tx_queue_size: 1024, tx_gas_limit: !U256::zero(), pending_set: PendingSet::SealingOrElseQueue, + reseal_min_period: Duration::from_secs(0), + work_queue_size: 50, }, spec, Some(accounts) From 98ae9cad6f042a69dafd53a27498f3fec7f7a0b0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 16:29:04 +0200 Subject: [PATCH 11/45] Minor additions to allow resetting of code. (#1482) * Minor additions to allow resetting of code. * Add test. --- ethcore/src/account.rs | 23 +++++++++++++++++++++-- ethcore/src/state.rs | 7 ++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ethcore/src/account.rs b/ethcore/src/account.rs index d0628436d..2edbf87ae 100644 --- a/ethcore/src/account.rs +++ b/ethcore/src/account.rs @@ -108,6 +108,12 @@ impl Account { self.code_cache = code; } + /// Reset this account's code to the given code. + pub fn reset_code(&mut self, code: Bytes) { + self.code_hash = None; + self.init_code(code); + } + /// Set (and cache) the contents of the trie's storage at `key` to `value`. pub fn set_storage(&mut self, key: H256, value: H256) { self.storage_overlay.borrow_mut().insert(key, (Filth::Dirty, value)); @@ -336,6 +342,21 @@ mod tests { assert_eq!(a.code_hash().hex(), "af231e631776a517ca23125370d542873eca1fb4d613ed9b5d5335a46ae5b7eb"); } + #[test] + fn reset_code() { + let mut a = Account::new_contract(69.into(), 0.into()); + let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); + a.init_code(vec![0x55, 0x44, 0xffu8]); + assert_eq!(a.code_hash(), SHA3_EMPTY); + a.commit_code(&mut db); + assert_eq!(a.code_hash().hex(), "af231e631776a517ca23125370d542873eca1fb4d613ed9b5d5335a46ae5b7eb"); + a.reset_code(vec![0x55]); + assert_eq!(a.code_hash(), SHA3_EMPTY); + a.commit_code(&mut db); + assert_eq!(a.code_hash().hex(), "37bf2238b11b68cdc8382cece82651b59d3c3988873b6e0f33d79694aa45f1be"); + } + #[test] fn rlpio() { let a = Account::new(U256::from(69u8), U256::from(0u8), HashMap::new(), Bytes::new()); @@ -348,7 +369,6 @@ mod tests { #[test] fn new_account() { - let a = Account::new(U256::from(69u8), U256::from(0u8), HashMap::new(), Bytes::new()); assert_eq!(a.rlp().to_hex(), "f8448045a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"); assert_eq!(a.balance(), &U256::from(69u8)); @@ -359,7 +379,6 @@ mod tests { #[test] fn create_account() { - let a = Account::new(U256::from(69u8), U256::from(0u8), HashMap::new(), Bytes::new()); assert_eq!(a.rlp().to_hex(), "f8448045a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"); } diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index e7981abba..0c086ffc3 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -208,12 +208,17 @@ impl State { self.require(a, false).set_storage(key, value) } - /// Initialise the code of account `a` so that it is `value` for `key`. + /// Initialise the code of account `a` so that it is `code`. /// NOTE: Account should have been created with `new_contract`. pub fn init_code(&mut self, a: &Address, code: Bytes) { self.require_or_from(a, true, || Account::new_contract(0.into(), self.account_start_nonce), |_|{}).init_code(code); } + /// Reset the code of account `a` so that it is `code`. + pub fn reset_code(&mut self, a: &Address, code: Bytes) { + self.require_or_from(a, true, || Account::new_contract(0.into(), self.account_start_nonce), |_|{}).reset_code(code); + } + /// Execute a given transaction. /// This will change the state accordingly. pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, vm_factory: &EvmFactory, t: &SignedTransaction, tracing: bool) -> ApplyResult { From aec4130dca5f1ca2f7cd36b92d718d0e83f9bd05 Mon Sep 17 00:00:00 2001 From: Adam Goldman Date: Wed, 29 Jun 2016 17:19:04 +0200 Subject: [PATCH 12/45] topbar dialog fix (#1479) * topbar dialog fix * push goldylucks changes to ethcore umbrella --- Cargo.lock | 6 +++--- dapps/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8448b0a97..87e66a516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,7 +284,7 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-dapps 0.3.0 (git+https://github.com/ethcore/parity-dapps-rs.git)", - "parity-dapps-builtins 0.5.1 (git+https://github.com/ethcore/parity-dapps-builtins-rs.git)", + "parity-dapps-builtins 0.5.2 (git+https://github.com/ethcore/parity-dapps-builtins-rs.git)", "parity-dapps-status 0.5.0 (git+https://github.com/ethcore/parity-dapps-status-rs.git)", "parity-dapps-wallet 0.6.1 (git+https://github.com/ethcore/parity-dapps-wallet-rs.git)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -909,8 +909,8 @@ dependencies = [ [[package]] name = "parity-dapps-builtins" -version = "0.5.1" -source = "git+https://github.com/ethcore/parity-dapps-builtins-rs.git#7408838e8ca3b57c6b0cf5da2e31e0e275959955" +version = "0.5.2" +source = "git+https://github.com/ethcore/parity-dapps-builtins-rs.git#01af2091d5d70dfe0aecbfd96308f0ae79fc61e6" dependencies = [ "parity-dapps 0.3.0 (git+https://github.com/ethcore/parity-dapps-rs.git)", ] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 0476e9919..6aaceb50a 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -23,7 +23,7 @@ ethcore-util = { path = "../util" } parity-dapps = { git = "https://github.com/ethcore/parity-dapps-rs.git", version = "0.3" } # List of apps parity-dapps-status = { git = "https://github.com/ethcore/parity-dapps-status-rs.git", version = "0.5.0" } -parity-dapps-builtins = { git = "https://github.com/ethcore/parity-dapps-builtins-rs.git", version = "0.5.0" } +parity-dapps-builtins = { git = "https://github.com/ethcore/parity-dapps-builtins-rs.git", version = "0.5.2" } parity-dapps-wallet = { git = "https://github.com/ethcore/parity-dapps-wallet-rs.git", version = "0.6.0", optional = true } parity-dapps-dao = { git = "https://github.com/ethcore/parity-dapps-dao-rs.git", version = "0.4.0", optional = true } parity-dapps-makerotc = { git = "https://github.com/ethcore/parity-dapps-makerotc-rs.git", version = "0.3.0", optional = true } From 5958c87e56dae73331d1336968de36a3d0bb6b52 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 29 Jun 2016 20:04:52 +0200 Subject: [PATCH 13/45] HTTP work notifier --- Cargo.lock | 7 ++++--- ethcore/Cargo.toml | 4 ++++ ethcore/src/ethereum/ethash.rs | 4 +--- ethcore/src/lib.rs | 2 ++ ethcore/src/miner/miner.rs | 36 +++++++++++++++++++++++++--------- ethcore/src/miner/mod.rs | 1 + parity/cli.rs | 2 +- parity/configuration.rs | 4 ++-- 8 files changed, 42 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8448b0a97..98a64d795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,6 +261,7 @@ dependencies = [ "ethjson 0.1.0", "ethstore 0.1.0", "heapsize 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -547,7 +548,7 @@ dependencies = [ [[package]] name = "hyper" version = "0.9.4" -source = "git+https://github.com/ethcore/hyper#7ccfcb2aa7e6aa6300efa8cebd6a0e6ce55582ea" +source = "git+https://github.com/ethcore/hyper#9e346c1d4bc30cd4142dea9d8a0b117d30858ca4" dependencies = [ "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -556,7 +557,7 @@ dependencies = [ "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rotor 0.6.3 (git+https://github.com/ethcore/rotor)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", - "spmc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spmc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1222,7 +1223,7 @@ dependencies = [ [[package]] name = "spmc" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 2b56bf581..2d94aebab 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -32,6 +32,10 @@ bloomchain = "0.1" rayon = "0.3.1" ethstore = { path = "../ethstore" } +[dependencies.hyper] +git = "https://github.com/ethcore/hyper" +default-features = false + [features] jit = ["evmjit"] evm-debug = [] diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 3400220db..84c2a9608 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -extern crate ethash; - -use self::ethash::{quick_get_difficulty, EthashManager, H256 as EH256}; +use ethash::{quick_get_difficulty, EthashManager, H256 as EH256}; use common::*; use block::*; use spec::CommonParams; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 9919ec62a..54a944331 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -91,6 +91,8 @@ extern crate ethjson; extern crate bloomchain; #[macro_use] extern crate ethcore_ipc as ipc; extern crate rayon; +extern crate hyper; +extern crate ethash; pub extern crate ethstore; #[cfg(test)] extern crate ethcore_devtools as devtools; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 8743edff3..7171e734b 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -28,6 +28,7 @@ use receipt::{Receipt}; use spec::Spec; use engine::Engine; use miner::{MinerService, MinerStatus, TransactionQueue, AccountDetails, TransactionImportResult, TransactionOrigin}; +use miner::work_notify::WorkPoster; /// Different possible definitions for pending transaction set. #[derive(Debug)] @@ -90,6 +91,7 @@ pub struct Miner { spec: Spec, accounts: Option>, + work_poster: Option, } impl Miner { @@ -106,14 +108,16 @@ impl Miner { extra_data: RwLock::new(Vec::new()), accounts: None, spec: spec, + work_poster: None, } } /// Creates new instance of miner pub fn new(options: MinerOptions, spec: Spec, accounts: Option>) -> Arc { + let work_poster = if !options.new_work_notify.is_empty() { Some(WorkPoster::new(&options.new_work_notify)) } else { None }; Arc::new(Miner { transaction_queue: Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, options.tx_gas_limit)), - sealing_enabled: AtomicBool::new(options.force_sealing), + sealing_enabled: AtomicBool::new(options.force_sealing || !options.new_work_notify.is_empty()), options: options, sealing_block_last_request: Mutex::new(0), sealing_work: Mutex::new(UsingQueue::new(5)), @@ -122,6 +126,7 @@ impl Miner { extra_data: RwLock::new(Vec::new()), accounts: accounts, spec: spec, + work_poster: work_poster, }) } @@ -129,6 +134,10 @@ impl Miner { self.spec.engine.deref() } + fn forced_sealing(&self) -> bool { + self.options.force_sealing || !self.options.new_work_notify.is_empty() + } + /// Prepares new block for sealing including top transactions from queue. #[cfg_attr(feature="dev", allow(match_same_arms))] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] @@ -230,13 +239,22 @@ impl Miner { } } - let mut sealing_work = self.sealing_work.lock().unwrap(); - if sealing_work.peek_last_ref().map_or(true, |pb| pb.block().fields().header.hash() != block.block().fields().header.hash()) { - trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); - sealing_work.push(block); - } - - trace!(target: "miner", "prepare_sealing: leaving (last={:?})", sealing_work.peek_last_ref().map(|b| b.block().fields().header.hash())); + let work = { + let mut sealing_work = self.sealing_work.lock().unwrap(); + let work = if sealing_work.peek_last_ref().map_or(true, |pb| pb.block().fields().header.hash() != block.block().fields().header.hash()) { + trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); + let pow_hash = block.block().fields().header.hash(); + let number = block.block().fields().header.number(); + let difficulty = *block.block().fields().header.difficulty(); + sealing_work.push(block); + Some((pow_hash, difficulty, number)) + } else { + None + }; + trace!(target: "miner", "prepare_sealing: leaving (last={:?})", sealing_work.peek_last_ref().map(|b| b.block().fields().header.hash())); + work + }; + work.map(|(pow_hash, difficulty, number)| self.work_poster.as_ref().map(|ref p| p.notify(pow_hash, difficulty, number))); } fn update_gas_limit(&self, chain: &MiningBlockChainClient) { @@ -552,7 +570,7 @@ impl MinerService for Miner { let current_no = chain.chain_info().best_block_number; let has_local_transactions = self.transaction_queue.lock().unwrap().has_local_pending_transactions(); let last_request = *self.sealing_block_last_request.lock().unwrap(); - let should_disable_sealing = !self.options.force_sealing + let should_disable_sealing = !self.forced_sealing() && !has_local_transactions && current_no > last_request && current_no - last_request > SEALING_TIMEOUT_IN_BLOCKS; diff --git a/ethcore/src/miner/mod.rs b/ethcore/src/miner/mod.rs index e65d6048a..152bd1a61 100644 --- a/ethcore/src/miner/mod.rs +++ b/ethcore/src/miner/mod.rs @@ -45,6 +45,7 @@ mod miner; mod external; mod transaction_queue; +mod work_notify; pub use self::transaction_queue::{TransactionQueue, AccountDetails, TransactionImportResult, TransactionOrigin}; pub use self::miner::{Miner, MinerOptions, PendingSet}; diff --git a/parity/cli.rs b/parity/cli.rs index a8e53eeff..752f6ce70 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -313,7 +313,7 @@ pub struct Args { pub flag_gas_cap: String, pub flag_extra_data: Option, pub flag_tx_queue_size: usize, - pub flag_work_notify: Option, + pub flag_notify_work: Option, pub flag_logging: Option, pub flag_version: bool, pub flag_from: String, diff --git a/parity/configuration.rs b/parity/configuration.rs index aebea8e61..d708ef02c 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -83,8 +83,8 @@ impl Configuration { ) } - pub fn work_notify(&self) -> Vec { - self.args.flag_work_notify.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) + fn work_notify(&self) -> Vec { + self.args.flag_notify_work.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) } pub fn miner_options(&self) -> MinerOptions { From e24f9c993604e8bdfb5117bd4138c9e5e05bcfe7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 15:37:11 +0200 Subject: [PATCH 14/45] Add CLI option and route to MinerOptions. --- ethcore/src/miner/miner.rs | 3 +++ parity/cli.rs | 3 +++ parity/configuration.rs | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 4518e416a..d91a13b96 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -45,6 +45,8 @@ pub enum PendingSet { /// Configures the behaviour of the miner. #[derive(Debug)] pub struct MinerOptions { + /// URLs to notify when there is new work. + pub new_work_notify: Vec, /// Force the miner to reseal, even when nobody has asked for work. pub force_sealing: bool, /// Reseal on receipt of new external transactions. @@ -66,6 +68,7 @@ pub struct MinerOptions { impl Default for MinerOptions { fn default() -> Self { MinerOptions { + new_work_notify: vec![], force_sealing: false, reseal_on_external_tx: true, reseal_on_own_tx: true, diff --git a/parity/cli.rs b/parity/cli.rs index a48ce3aa7..e91acc5c6 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -169,6 +169,8 @@ Sealing/Mining Options: more than 32 characters. --tx-queue-size LIMIT Maximum amount of transactions in the queue (waiting to be included in next block) [default: 1024]. + --notify-work URLS URLs to which work package notifications are pushed. + URLS should be a comma-delimited list of HTTP URLs. Footprint Options: --tracing BOOL Indicates if full transaction tracing should be @@ -320,6 +322,7 @@ pub struct Args { pub flag_gas_cap: String, pub flag_extra_data: Option, pub flag_tx_queue_size: usize, + pub flag_work_notify: Option, pub flag_logging: Option, pub flag_version: bool, pub flag_from: String, diff --git a/parity/configuration.rs b/parity/configuration.rs index e95ef4233..ffdf44126 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -84,6 +84,10 @@ impl Configuration { ) } + pub fn work_notify(&self) -> Vec { + self.args.flag_work_notify.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) + } + pub fn miner_options(&self) -> MinerOptions { let (own, ext) = match self.args.flag_reseal_on_txs.as_str() { "none" => (false, false), @@ -93,6 +97,7 @@ impl Configuration { x => die!("{}: Invalid value for --reseal option. Use --help for more information.", x) }; MinerOptions { + new_work_notify: self.work_notify(), force_sealing: self.args.flag_force_sealing, reseal_on_external_tx: ext, reseal_on_own_tx: own, From b3f37f3cb46300ee2a10e790b8fb22c1902db7d3 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 29 Jun 2016 20:07:21 +0200 Subject: [PATCH 15/45] HTTP work notifier --- Cargo.lock | 7 ++- ethcore/Cargo.toml | 4 ++ ethcore/src/ethereum/ethash.rs | 4 +- ethcore/src/lib.rs | 2 + ethcore/src/miner/miner.rs | 37 ++++++++--- ethcore/src/miner/mod.rs | 1 + ethcore/src/miner/work_notify.rs | 105 +++++++++++++++++++++++++++++++ parity/cli.rs | 2 +- parity/configuration.rs | 4 +- 9 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 ethcore/src/miner/work_notify.rs diff --git a/Cargo.lock b/Cargo.lock index 87e66a516..0a2480ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,6 +261,7 @@ dependencies = [ "ethjson 0.1.0", "ethstore 0.1.0", "heapsize 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.9.4 (git+https://github.com/ethcore/hyper)", "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -547,7 +548,7 @@ dependencies = [ [[package]] name = "hyper" version = "0.9.4" -source = "git+https://github.com/ethcore/hyper#7ccfcb2aa7e6aa6300efa8cebd6a0e6ce55582ea" +source = "git+https://github.com/ethcore/hyper#9e346c1d4bc30cd4142dea9d8a0b117d30858ca4" dependencies = [ "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -556,7 +557,7 @@ dependencies = [ "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rotor 0.6.3 (git+https://github.com/ethcore/rotor)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", - "spmc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spmc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1222,7 +1223,7 @@ dependencies = [ [[package]] name = "spmc" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 2b56bf581..2d94aebab 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -32,6 +32,10 @@ bloomchain = "0.1" rayon = "0.3.1" ethstore = { path = "../ethstore" } +[dependencies.hyper] +git = "https://github.com/ethcore/hyper" +default-features = false + [features] jit = ["evmjit"] evm-debug = [] diff --git a/ethcore/src/ethereum/ethash.rs b/ethcore/src/ethereum/ethash.rs index 3400220db..84c2a9608 100644 --- a/ethcore/src/ethereum/ethash.rs +++ b/ethcore/src/ethereum/ethash.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -extern crate ethash; - -use self::ethash::{quick_get_difficulty, EthashManager, H256 as EH256}; +use ethash::{quick_get_difficulty, EthashManager, H256 as EH256}; use common::*; use block::*; use spec::CommonParams; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 9919ec62a..54a944331 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -91,6 +91,8 @@ extern crate ethjson; extern crate bloomchain; #[macro_use] extern crate ethcore_ipc as ipc; extern crate rayon; +extern crate hyper; +extern crate ethash; pub extern crate ethstore; #[cfg(test)] extern crate ethcore_devtools as devtools; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index d91a13b96..18202b1f2 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -29,6 +29,7 @@ use receipt::{Receipt}; use spec::Spec; use engine::Engine; use miner::{MinerService, MinerStatus, TransactionQueue, AccountDetails, TransactionImportResult, TransactionOrigin}; +use miner::work_notify::WorkPoster; /// Different possible definitions for pending transaction set. #[derive(Debug)] @@ -98,6 +99,7 @@ pub struct Miner { spec: Spec, accounts: Option>, + work_poster: Option, } impl Miner { @@ -115,15 +117,18 @@ impl Miner { extra_data: RwLock::new(Vec::new()), accounts: None, spec: spec, + work_poster: None, } } /// Creates new instance of miner pub fn new(options: MinerOptions, spec: Spec, accounts: Option>) -> Arc { + let work_poster = if !options.new_work_notify.is_empty() { Some(WorkPoster::new(&options.new_work_notify)) } else { None }; Arc::new(Miner { transaction_queue: Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, options.tx_gas_limit)), - sealing_enabled: AtomicBool::new(options.force_sealing), + sealing_enabled: AtomicBool::new(options.force_sealing || !options.new_work_notify.is_empty()), next_allowed_reseal: Mutex::new(Instant::now()), + options: options, sealing_block_last_request: Mutex::new(0), sealing_work: Mutex::new(UsingQueue::new(options.work_queue_size)), gas_range_target: RwLock::new((U256::zero(), U256::zero())), @@ -132,6 +137,7 @@ impl Miner { accounts: accounts, options: options, spec: spec, + work_poster: work_poster, }) } @@ -139,6 +145,10 @@ impl Miner { self.spec.engine.deref() } + fn forced_sealing(&self) -> bool { + self.options.force_sealing || !self.options.new_work_notify.is_empty() + } + /// Prepares new block for sealing including top transactions from queue. #[cfg_attr(feature="dev", allow(match_same_arms))] #[cfg_attr(feature="dev", allow(cyclomatic_complexity))] @@ -240,13 +250,22 @@ impl Miner { } } - let mut sealing_work = self.sealing_work.lock().unwrap(); - if sealing_work.peek_last_ref().map_or(true, |pb| pb.block().fields().header.hash() != block.block().fields().header.hash()) { - trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); - sealing_work.push(block); - } - - trace!(target: "miner", "prepare_sealing: leaving (last={:?})", sealing_work.peek_last_ref().map(|b| b.block().fields().header.hash())); + let work = { + let mut sealing_work = self.sealing_work.lock().unwrap(); + let work = if sealing_work.peek_last_ref().map_or(true, |pb| pb.block().fields().header.hash() != block.block().fields().header.hash()) { + trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); + let pow_hash = block.block().fields().header.hash(); + let number = block.block().fields().header.number(); + let difficulty = *block.block().fields().header.difficulty(); + sealing_work.push(block); + Some((pow_hash, difficulty, number)) + } else { + None + }; + trace!(target: "miner", "prepare_sealing: leaving (last={:?})", sealing_work.peek_last_ref().map(|b| b.block().fields().header.hash())); + work + }; + work.map(|(pow_hash, difficulty, number)| self.work_poster.as_ref().map(|ref p| p.notify(pow_hash, difficulty, number))); } fn update_gas_limit(&self, chain: &MiningBlockChainClient) { @@ -565,7 +584,7 @@ impl MinerService for Miner { let current_no = chain.chain_info().best_block_number; let has_local_transactions = self.transaction_queue.lock().unwrap().has_local_pending_transactions(); let last_request = *self.sealing_block_last_request.lock().unwrap(); - let should_disable_sealing = !self.options.force_sealing + let should_disable_sealing = !self.forced_sealing() && !has_local_transactions && current_no > last_request && current_no - last_request > SEALING_TIMEOUT_IN_BLOCKS; diff --git a/ethcore/src/miner/mod.rs b/ethcore/src/miner/mod.rs index e65d6048a..152bd1a61 100644 --- a/ethcore/src/miner/mod.rs +++ b/ethcore/src/miner/mod.rs @@ -45,6 +45,7 @@ mod miner; mod external; mod transaction_queue; +mod work_notify; pub use self::transaction_queue::{TransactionQueue, AccountDetails, TransactionImportResult, TransactionOrigin}; pub use self::miner::{Miner, MinerOptions, PendingSet}; diff --git a/ethcore/src/miner/work_notify.rs b/ethcore/src/miner/work_notify.rs new file mode 100644 index 000000000..6144e2d3d --- /dev/null +++ b/ethcore/src/miner/work_notify.rs @@ -0,0 +1,105 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +extern crate hyper; + +use hyper::header::ContentType; +use hyper::method::Method; +use hyper::client::{Request, Response, Client}; +use hyper::{Next}; +use hyper::net::HttpStream; +use ethash::SeedHashCompute; +use hyper::Url; +use util::*; +use ethereum::ethash::Ethash; + +pub struct WorkPoster { + urls: Vec, + client: Mutex>, + seed_compute: Mutex, +} + +impl WorkPoster { + pub fn new(urls: &[String]) -> Self { + let urls = urls.into_iter().filter_map(|u| { + match Url::parse(&u) { + Ok(url) => Some(url), + Err(e) => { + warn!("Error parsing URL {} : {}", u, e); + None + } + } + }).collect(); + let client = Client::::configure() + .keep_alive(false) + .build().expect("Error creating HTTP client"); + WorkPoster { + client: Mutex::new(client), + urls: urls, + seed_compute: Mutex::new(SeedHashCompute::new()), + } + } + + pub fn notify(&self, pow_hash: H256, difficulty: U256, number: u64) { + // TODO: move this to engine + let target = Ethash::difficulty_to_boundary(&difficulty); + let seed_hash = &self.seed_compute.lock().unwrap().get_seedhash(number); + let seed_hash = H256::from_slice(&seed_hash[..]); + let body = format!(r#"{{ "result": ["0x{}","0x{}","0x{}","0x{:x}"] }}"#, + pow_hash.hex(), seed_hash.hex(), target.hex(), number); + let client = self.client.lock().unwrap(); + for u in &self.urls { + if let Err(e) = client.request(u.clone(), PostHandler { body: body.clone() }) { + warn!("Error sending HTTP notification to {} : {}", u, e); + } + } + } +} + +struct PostHandler { + body: String, +} + +impl hyper::client::Handler for PostHandler { + fn on_request(&mut self, request: &mut Request) -> Next { + request.set_method(Method::Post); + request.headers_mut().set(ContentType::json()); + Next::write() + } + + fn on_request_writable(&mut self, encoder: &mut hyper::Encoder) -> Next { + if let Err(e) = encoder.write_all(self.body.as_bytes()) { + trace!("Error posting work data: {}", e); + } + encoder.close(); + Next::read() + + } + + fn on_response(&mut self, _response: Response) -> Next { + Next::end() + } + + fn on_response_readable(&mut self, _decoder: &mut hyper::Decoder) -> Next { + Next::end() + } + + fn on_error(&mut self, err: hyper::Error) -> Next { + trace!("Error posting work data: {}", err); + Next::end() + } +} + diff --git a/parity/cli.rs b/parity/cli.rs index e91acc5c6..3c1eee8e8 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -322,7 +322,7 @@ pub struct Args { pub flag_gas_cap: String, pub flag_extra_data: Option, pub flag_tx_queue_size: usize, - pub flag_work_notify: Option, + pub flag_notify_work: Option, pub flag_logging: Option, pub flag_version: bool, pub flag_from: String, diff --git a/parity/configuration.rs b/parity/configuration.rs index ffdf44126..0cfb7c44f 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -84,8 +84,8 @@ impl Configuration { ) } - pub fn work_notify(&self) -> Vec { - self.args.flag_work_notify.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) + fn work_notify(&self) -> Vec { + self.args.flag_notify_work.as_ref().map_or_else(Vec::new, |s| s.split(',').map(|s| s.to_owned()).collect()) } pub fn miner_options(&self) -> MinerOptions { From dc244489009a32e3f03f36014dabc19f64fdeb0c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 15:43:34 +0200 Subject: [PATCH 16/45] Include number in eth_getWork. --- ethcore/src/miner/miner.rs | 3 +-- rpc/src/v1/impls/eth.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 18202b1f2..e539c7f0e 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -128,14 +128,13 @@ impl Miner { transaction_queue: Mutex::new(TransactionQueue::with_limits(options.tx_queue_size, options.tx_gas_limit)), sealing_enabled: AtomicBool::new(options.force_sealing || !options.new_work_notify.is_empty()), next_allowed_reseal: Mutex::new(Instant::now()), - options: options, sealing_block_last_request: Mutex::new(0), sealing_work: Mutex::new(UsingQueue::new(options.work_queue_size)), + options: options, gas_range_target: RwLock::new((U256::zero(), U256::zero())), author: RwLock::new(Address::default()), extra_data: RwLock::new(Vec::new()), accounts: accounts, - options: options, spec: spec, work_poster: work_poster, }) diff --git a/rpc/src/v1/impls/eth.rs b/rpc/src/v1/impls/eth.rs index 05dc89564..8bfc661e3 100644 --- a/rpc/src/v1/impls/eth.rs +++ b/rpc/src/v1/impls/eth.rs @@ -498,7 +498,7 @@ impl Eth for EthClient where let pow_hash = b.hash(); let target = Ethash::difficulty_to_boundary(b.block().header().difficulty()); let seed_hash = &self.seed_compute.lock().unwrap().get_seedhash(b.block().header().number()); - to_value(&(pow_hash, H256::from_slice(&seed_hash[..]), target)) + to_value(&(pow_hash, H256::from_slice(&seed_hash[..]), target, &U256::from(b.block().header().number()))) }).unwrap_or(Err(Error::internal_error())) // no work found. }, _ => Err(Error::invalid_params()) From ee01ad132426b350dd238370401a8b6f868ea078 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 29 Jun 2016 21:49:12 +0200 Subject: [PATCH 17/45] Shortcut sealed block into the chain --- ethcore/src/block.rs | 14 +++- ethcore/src/client/client.rs | 114 ++++++++++++++++++++---------- ethcore/src/client/mod.rs | 5 +- ethcore/src/client/test_client.rs | 6 +- ethcore/src/miner/miner.rs | 17 +++-- ethcore/src/service.rs | 2 + ethcore/src/tests/helpers.rs | 2 +- sync/src/chain.rs | 69 +++++++++++++----- sync/src/lib.rs | 4 +- sync/src/tests/helpers.rs | 2 +- 10 files changed, 163 insertions(+), 72 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 103435f40..13a2024d9 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -164,6 +164,12 @@ pub trait IsBlock { fn uncles(&self) -> &Vec
{ &self.block().base.uncles } } +/// Trait for a object that has a state database. +pub trait Drain { + /// Drop this object and return the underlieing database. + fn drain(self) -> Box; +} + impl IsBlock for ExecutedBlock { fn block(&self) -> &ExecutedBlock { self } } @@ -436,9 +442,11 @@ impl LockedBlock { _ => Ok(SealedBlock { block: s.block, uncle_bytes: s.uncle_bytes }), } } +} +impl Drain for LockedBlock { /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box { self.block.state.drop().1 } + fn drain(self) -> Box { self.block.state.drop().1 } } impl SealedBlock { @@ -450,9 +458,11 @@ impl SealedBlock { block_rlp.append_raw(&self.uncle_bytes, 1); block_rlp.out() } +} +impl Drain for SealedBlock { /// Drop this object and return the underlieing database. - pub fn drain(self) -> Box { self.block.state.drop().1 } + fn drain(self) -> Box { self.block.state.drop().1 } } impl IsBlock for SealedBlock { diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 5d157b654..3cc004d20 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -249,7 +249,7 @@ impl Client { Ok(locked_block) } - fn calculate_enacted_retracted(&self, import_results: Vec) -> (Vec, Vec) { + fn calculate_enacted_retracted(&self, import_results: &[ImportRoute]) -> (Vec, Vec) { fn map_to_vec(map: Vec<(H256, bool)>) -> Vec { map.into_iter().map(|(k, _v)| k).collect() } @@ -259,12 +259,12 @@ impl Client { // could be retracted in import `k+1`. This is why to understand if after all inserts // the block is enacted or retracted we iterate over all routes and at the end final state // will be in the hashmap - let map = import_results.into_iter().fold(HashMap::new(), |mut map, route| { - for hash in route.enacted { - map.insert(hash, true); + let map = import_results.iter().fold(HashMap::new(), |mut map, route| { + for hash in &route.enacted { + map.insert(hash.clone(), true); } - for hash in route.retracted { - map.insert(hash, false); + for hash in &route.retracted { + map.insert(hash.clone(), false); } map }); @@ -301,36 +301,10 @@ impl Client { invalid_blocks.insert(header.hash()); continue; } + let closed_block = closed_block.unwrap(); imported_blocks.push(header.hash()); - // Are we committing an era? - let ancient = if header.number() >= HISTORY { - let n = header.number() - HISTORY; - Some((n, self.chain.block_hash(n).unwrap())) - } else { - None - }; - - // Commit results - let closed_block = closed_block.unwrap(); - let receipts = closed_block.block().receipts().clone(); - let traces = From::from(closed_block.block().traces().clone().unwrap_or_else(Vec::new)); - - closed_block.drain() - .commit(header.number(), &header.hash(), ancient) - .expect("State DB commit failed."); - - // And update the chain after commit to prevent race conditions - // (when something is in chain but you are not able to fetch details) - let route = self.chain.insert_block(&block.bytes, receipts); - self.tracedb.import(TraceImportRequest { - traces: traces, - block_hash: header.hash(), - block_number: header.number(), - enacted: route.enacted.clone(), - retracted: route.retracted.len() - }); - + let route = self.commit_block(closed_block, &header.hash(), &block.bytes); import_results.push(route); self.report.write().unwrap().accrue_block(&block); @@ -351,7 +325,7 @@ impl Client { { if !imported_blocks.is_empty() && self.block_queue.queue_info().is_empty() { - let (enacted, retracted) = self.calculate_enacted_retracted(import_results); + let (enacted, retracted) = self.calculate_enacted_retracted(&import_results); if self.queue_info().is_empty() { self.miner.chain_new_blocks(self, &imported_blocks, &invalid_blocks, &enacted, &retracted); @@ -362,19 +336,47 @@ impl Client { invalid: invalid_blocks, enacted: enacted, retracted: retracted, + sealed: Vec::new(), })).unwrap_or_else(|e| warn!("Error sending IO notification: {:?}", e)); } } - { - if self.chain_info().best_block_hash != original_best { - self.miner.update_sealing(self); - } + if self.chain_info().best_block_hash != original_best { + self.miner.update_sealing(self); } imported } + fn commit_block(&self, block: B, hash: &H256, block_data: &Bytes) -> ImportRoute where B: IsBlock + Drain { + let number = block.header().number(); + // Are we committing an era? + let ancient = if number >= HISTORY { + let n = number - HISTORY; + Some((n, self.chain.block_hash(n).unwrap())) + } else { + None + }; + + // Commit results + let receipts = block.receipts().clone(); + let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new)); + + block.drain().commit(number, hash, ancient).expect("State DB commit failed."); + + // And update the chain after commit to prevent race conditions + // (when something is in chain but you are not able to fetch details) + let route = self.chain.insert_block(block_data, receipts); + self.tracedb.import(TraceImportRequest { + traces: traces, + block_hash: hash.clone(), + block_number: number, + enacted: route.enacted.clone(), + retracted: route.retracted.len() + }); + route + } + /// Import transactions from the IO queue pub fn import_queued_transactions(&self, transactions: &[Bytes]) -> usize { let _timer = PerfTimer::new("import_queued_transactions"); @@ -830,6 +832,40 @@ impl MiningBlockChainClient for Client { fn vm_factory(&self) -> &EvmFactory { &self.vm_factory } + + fn import_sealed_block(&self, block: SealedBlock) -> ImportResult { + let _import_lock = self.import_lock.lock(); + let _timer = PerfTimer::new("import_sealed_block"); + + let original_best = self.chain_info().best_block_hash; + + let h = block.header().hash(); + let number = block.header().number(); + + let block_data = block.rlp_bytes(); + let route = self.commit_block(block, &h, &block_data); + trace!(target: "client", "Imported sealed block #{} ({})", number, h); + + { + let (enacted, retracted) = self.calculate_enacted_retracted(&[route]); + self.miner.chain_new_blocks(self, &[h.clone()], &[], &enacted, &retracted); + + self.io_channel.send(NetworkIoMessage::User(SyncMessage::NewChainBlocks { + imported: vec![h.clone()], + invalid: vec![], + enacted: enacted, + retracted: retracted, + sealed: vec![h.clone()], + })).unwrap_or_else(|e| warn!("Error sending IO notification: {:?}", e)); + } + + if self.chain_info().best_block_hash != original_best { + self.miner.update_sealing(self); + } + + info!("Block {} ({}) submitted and imported.", h.hex(), number); + Ok(h) + } } impl MayPanic for Client { diff --git a/ethcore/src/client/mod.rs b/ethcore/src/client/mod.rs index bef814b4e..7f3c3bb3a 100644 --- a/ethcore/src/client/mod.rs +++ b/ethcore/src/client/mod.rs @@ -37,7 +37,7 @@ use util::numbers::U256; use util::Itertools; use blockchain::TreeRoute; use block_queue::BlockQueueInfo; -use block::OpenBlock; +use block::{OpenBlock, SealedBlock}; use header::{BlockNumber, Header}; use transaction::{LocalizedTransaction, SignedTransaction}; use log_entry::LocalizedLogEntry; @@ -253,4 +253,7 @@ pub trait MiningBlockChainClient : BlockChainClient { /// Returns EvmFactory. fn vm_factory(&self) -> &EvmFactory; + + /// Import sealed block. Skips all verifications. + fn import_sealed_block(&self, block: SealedBlock) -> ImportResult; } diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index ed1f10e09..f51f978de 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -32,7 +32,7 @@ use miner::{Miner, MinerService}; use spec::Spec; use block_queue::BlockQueueInfo; -use block::OpenBlock; +use block::{OpenBlock, SealedBlock}; use executive::Executed; use error::{ExecutionError}; use trace::LocalizedTrace; @@ -248,6 +248,10 @@ impl MiningBlockChainClient for TestBlockChainClient { fn vm_factory(&self) -> &EvmFactory { unimplemented!(); } + + fn import_sealed_block(&self, _block: SealedBlock) -> ImportResult { + unimplemented!(); + } } impl BlockChainClient for TestBlockChainClient { diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 4518e416a..30691519d 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -589,26 +589,25 @@ impl MinerService for Miner { } fn submit_seal(&self, chain: &MiningBlockChainClient, pow_hash: H256, seal: Vec) -> Result<(), Error> { - if let Some(b) = self.sealing_work.lock().unwrap().take_used_if(|b| &b.hash() == &pow_hash) { + let result = if let Some(b) = self.sealing_work.lock().unwrap().take_used_if(|b| &b.hash() == &pow_hash) { match b.lock().try_seal(self.engine(), seal) { Err(_) => { info!(target: "miner", "Mined block rejected, PoW was invalid."); Err(Error::PowInvalid) } Ok(sealed) => { - info!(target: "miner", "New block mined, hash: {}", sealed.header().hash()); - // TODO: commit DB from `sealed.drain` and make a VerifiedBlock to skip running the transactions twice. - let b = sealed.rlp_bytes(); - let h = b.sha3(); - try!(chain.import_block(b)); - info!("Block {} submitted and imported.", h); - Ok(()) + info!(target: "miner", "New block mined, hash: {}", sealed.header().hash().hex()); + Ok(sealed) } } } else { info!(target: "miner", "Mined block rejected, PoW hash invalid or out of date."); Err(Error::PowHashInvalid) - } + }; + result.and_then(|sealed| { + try!(chain.import_sealed_block(sealed)); + Ok(()) + }) } fn chain_new_blocks(&self, chain: &MiningBlockChainClient, _imported: &[H256], _invalid: &[H256], enacted: &[H256], retracted: &[H256]) { diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index 98fb3ad23..d8233a4b6 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -36,6 +36,8 @@ pub enum SyncMessage { retracted: Vec, /// Hashes of blocks that are now included in cannonical chain enacted: Vec, + /// Hashes of blocks that are sealed by this node + sealed: Vec, }, /// Best Block Hash in chain has been changed NewChainHead, diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index 15b346919..70a644896 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -17,7 +17,7 @@ use client::{BlockChainClient, Client, ClientConfig}; use common::*; use spec::*; -use block::{OpenBlock}; +use block::{OpenBlock, Drain}; use blockchain::{BlockChain, Config as BlockChainConfig}; use state::*; use evm::Schedule; diff --git a/sync/src/chain.rs b/sync/src/chain.rs index aa3657419..1897b02f4 100644 --- a/sync/src/chain.rs +++ b/sync/src/chain.rs @@ -1231,6 +1231,14 @@ impl ChainSync { rlp_stream.out() } + /// creates latest block rlp for the given client + fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { + let mut rlp_stream = RlpStream::new_list(2); + rlp_stream.append_raw(&chain.block(BlockID::Hash(hash.clone())).expect("Block has just been sealed; qed"), 1); + rlp_stream.append(&chain.block_total_difficulty(BlockID::Hash(hash.clone())).expect("Block has just been sealed; qed.")); + rlp_stream.out() + } + /// returns peer ids that have less blocks than our chain fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo, io: &SyncIo) -> Vec<(PeerId, BlockNumber)> { let latest_hash = chain_info.best_block_hash; @@ -1250,7 +1258,6 @@ impl ChainSync { .collect::>() } - fn select_lagging_peers(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> Vec<(PeerId, BlockNumber)> { use rand::Rng; let mut lagging_peers = self.get_lagging_peers(chain_info, io); @@ -1263,13 +1270,24 @@ impl ChainSync { } /// propagates latest block to lagging peers - fn propagate_blocks(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo) -> usize { - let lucky_peers = self.select_lagging_peers(chain_info, io); + fn propagate_blocks(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo, sealed: &[H256]) -> usize { + let lucky_peers: Vec<_> = if sealed.is_empty() { + self.select_lagging_peers(chain_info, io).iter().map(|&(id, _)| id).collect() + } else { + self.peers.keys().cloned().collect() + }; trace!(target: "sync", "Sending NewBlocks to {:?}", lucky_peers); let mut sent = 0; - for (peer_id, _) in lucky_peers { - let rlp = ChainSync::create_latest_block_rlp(io.chain()); - self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); + for peer_id in lucky_peers { + if sealed.is_empty() { + let rlp = ChainSync::create_latest_block_rlp(io.chain()); + self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); + } else { + for h in sealed { + let rlp = ChainSync::create_new_block_rlp(io.chain(), h); + self.send_packet(io, peer_id, NEW_BLOCK_PACKET, rlp); + } + } self.peers.get_mut(&peer_id).unwrap().latest_hash = chain_info.best_block_hash.clone(); self.peers.get_mut(&peer_id).unwrap().latest_number = Some(chain_info.best_block_number); sent += 1; @@ -1346,11 +1364,11 @@ impl ChainSync { sent } - fn propagate_latest_blocks(&mut self, io: &mut SyncIo) { + fn propagate_latest_blocks(&mut self, io: &mut SyncIo, sealed: &[H256]) { let chain_info = io.chain().chain_info(); if (((chain_info.best_block_number as i64) - (self.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { let hashes = self.propagate_new_hashes(&chain_info, io); - let blocks = self.propagate_blocks(&chain_info, io); + let blocks = self.propagate_blocks(&chain_info, io, sealed); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); } @@ -1365,10 +1383,10 @@ impl ChainSync { } /// called when block is imported to chain, updates transactions queue and propagates the blocks - pub fn chain_new_blocks(&mut self, io: &mut SyncIo, _imported: &[H256], invalid: &[H256], _enacted: &[H256], _retracted: &[H256]) { + pub fn chain_new_blocks(&mut self, io: &mut SyncIo, _imported: &[H256], invalid: &[H256], _enacted: &[H256], _retracted: &[H256], sealed: &[H256]) { if io.is_chain_queue_empty() { // Propagate latests blocks - self.propagate_latest_blocks(io); + self.propagate_latest_blocks(io, sealed); } if !invalid.is_empty() { trace!(target: "sync", "Bad blocks in the queue, restarting"); @@ -1637,7 +1655,26 @@ mod tests { let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - let peer_count = sync.propagate_blocks(&chain_info, &mut io); + let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[]); + + // 1 message should be send + assert_eq!(1, io.queue.len()); + // 1 peer should be updated + assert_eq!(1, peer_count); + // NEW_BLOCK_PACKET + assert_eq!(0x07, io.queue[0].packet_id); + } + + #[test] + fn sends_sealed_block() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + let mut queue = VecDeque::new(); + let hash = client.block_hash(BlockID::Number(99)).unwrap(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let chain_info = client.chain_info(); + let mut io = TestIo::new(&mut client, &mut queue, None); + let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[hash.clone()]); // 1 message should be send assert_eq!(1, io.queue.len()); @@ -1761,7 +1798,7 @@ mod tests { let chain_info = client.chain_info(); let mut io = TestIo::new(&mut client, &mut queue, None); - sync.propagate_blocks(&chain_info, &mut io); + sync.propagate_blocks(&chain_info, &mut io, &[]); let data = &io.queue[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &UntrustedRlp::new(data)); @@ -1794,7 +1831,7 @@ mod tests { let mut queue = VecDeque::new(); let mut io = TestIo::new(&mut client, &mut queue, None); io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks); - sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks); + sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]); assert_eq!(io.chain.miner.status().transactions_in_future_queue, 0); assert_eq!(io.chain.miner.status().transactions_in_pending_queue, 1); } @@ -1808,7 +1845,7 @@ mod tests { let mut queue = VecDeque::new(); let mut io = TestIo::new(&mut client, &mut queue, None); io.chain.miner.chain_new_blocks(io.chain, &[], &[], &good_blocks, &retracted_blocks); - sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks); + sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks, &[]); } // then @@ -1833,10 +1870,10 @@ mod tests { let mut io = TestIo::new(&mut client, &mut queue, None); // when - sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks); + sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[]); assert_eq!(io.chain.miner.status().transactions_in_future_queue, 0); assert_eq!(io.chain.miner.status().transactions_in_pending_queue, 0); - sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks); + sync.chain_new_blocks(&mut io, &[], &[], &good_blocks, &retracted_blocks, &[]); // then let status = io.chain.miner.status(); diff --git a/sync/src/lib.rs b/sync/src/lib.rs index 9bd10cb95..fa26e7d85 100644 --- a/sync/src/lib.rs +++ b/sync/src/lib.rs @@ -196,9 +196,9 @@ impl NetworkProtocolHandler for EthSync { #[cfg_attr(feature="dev", allow(single_match))] fn message(&self, io: &NetworkContext, message: &SyncMessage) { match *message { - SyncMessage::NewChainBlocks { ref imported, ref invalid, ref enacted, ref retracted } => { + SyncMessage::NewChainBlocks { ref imported, ref invalid, ref enacted, ref retracted, ref sealed } => { let mut sync_io = NetSyncIo::new(io, self.chain.deref()); - self.sync.write().unwrap().chain_new_blocks(&mut sync_io, imported, invalid, enacted, retracted); + self.sync.write().unwrap().chain_new_blocks(&mut sync_io, imported, invalid, enacted, retracted, sealed); }, _ => {/* Ignore other messages */}, } diff --git a/sync/src/tests/helpers.rs b/sync/src/tests/helpers.rs index 831976048..9a9afca49 100644 --- a/sync/src/tests/helpers.rs +++ b/sync/src/tests/helpers.rs @@ -173,6 +173,6 @@ impl TestNet { pub fn trigger_chain_new_blocks(&mut self, peer_id: usize) { let mut peer = self.peer_mut(peer_id); - peer.sync.write().unwrap().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None), &[], &[], &[], &[]); + peer.sync.write().unwrap().chain_new_blocks(&mut TestIo::new(&mut peer.chain, &mut peer.queue, None), &[], &[], &[], &[], &[]); } } From 86ba6f1912d78eb1a428553c704b9ea59d0c7b3e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 22:05:00 +0200 Subject: [PATCH 18/45] Fix test compilation. --- ethcore/src/miner/miner.rs | 1 - rpc/src/v1/tests/eth.rs | 1 + util/src/network/tests.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index f99fe5fac..aa08459f1 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -130,7 +130,6 @@ impl Miner { next_allowed_reseal: Mutex::new(Instant::now()), sealing_block_last_request: Mutex::new(0), sealing_work: Mutex::new(UsingQueue::new(options.work_queue_size)), - options: options, gas_range_target: RwLock::new((U256::zero(), U256::zero())), author: RwLock::new(Address::default()), extra_data: RwLock::new(Vec::new()), diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 59d06e84a..4eb80b1c0 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -60,6 +60,7 @@ fn miner_service(spec: Spec, accounts: Arc) -> Arc { pending_set: PendingSet::SealingOrElseQueue, reseal_min_period: Duration::from_secs(0), work_queue_size: 50, + new_work_notify: vec![], }, spec, Some(accounts) diff --git a/util/src/network/tests.rs b/util/src/network/tests.rs index 861edc144..cd3f48d9a 100644 --- a/util/src/network/tests.rs +++ b/util/src/network/tests.rs @@ -88,7 +88,7 @@ impl NetworkProtocolHandler for TestProtocol { /// Timer function called after a timeout created with `NetworkContext::timeout`. fn timeout(&self, io: &NetworkContext, timer: TimerToken) { - io.message(TestProtocolMessage { payload: 22 }); + io.message(TestProtocolMessage { payload: 22 }).unwrap(); assert_eq!(timer, 0); self.got_timeout.store(true, AtomicOrdering::Relaxed); } From 4a6206c5146ac141df98f0a92a45bbd6edccafda Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 16:45:17 +0200 Subject: [PATCH 19/45] Log for when we mine a block with lots of info. Fixes #1468 --- ethcore/src/miner/miner.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index aea0fd154..f1bba6fdf 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -617,7 +617,7 @@ impl MinerService for Miner { Err(Error::PowInvalid) } Ok(sealed) => { - info!(target: "miner", "New block mined, hash: {}", sealed.header().hash().hex()); + info!(target: "miner", "New solution received for #{}: {}", sealed.header().number(), sealed.header().hash()); Ok(sealed) } } @@ -626,7 +626,10 @@ impl MinerService for Miner { Err(Error::PowHashInvalid) }; result.and_then(|sealed| { + let n = sealed.header().number(); + let h = sealed.header().hash(); try!(chain.import_sealed_block(sealed)); + info!("Mined block imported OK. #{}: {}", n, h.hex()); Ok(()) }) } From 92edf7f511a7b8a07c273e1b8a1b4d87c603e1b5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 17:16:58 +0200 Subject: [PATCH 20/45] Safe coloured logging. --- Cargo.lock | 1 + ethcore/src/miner/miner.rs | 7 ++++--- parity/main.rs | 6 +++++- parity/setup_log.rs | 6 +++--- rpc/src/v1/tests/mocked/ethcore.rs | 2 +- util/Cargo.toml | 1 + util/src/lib.rs | 1 + util/src/log.rs | 19 +++++++++++++++++-- 8 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a2480ac1..18168f795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,6 +377,7 @@ dependencies = [ name = "ethcore-util" version = "1.3.0" dependencies = [ + "ansi_term 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "arrayvec 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", "bigint 0.1.0", "chrono 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index f1bba6fdf..8b6e9b49e 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -19,6 +19,7 @@ use std::sync::atomic::AtomicBool; use std::time::{Instant, Duration}; use util::*; +use util::Colour::White; use account_provider::AccountProvider; use views::{BlockView, HeaderView}; use client::{MiningBlockChainClient, Executive, Executed, EnvInfo, TransactOptions, BlockID, CallAnalytics}; @@ -613,7 +614,7 @@ impl MinerService for Miner { let result = if let Some(b) = self.sealing_work.lock().unwrap().take_used_if(|b| &b.hash() == &pow_hash) { match b.lock().try_seal(self.engine(), seal) { Err(_) => { - info!(target: "miner", "Mined block rejected, PoW was invalid."); + info!(target: "miner", "Mined solution rejected: Invalid."); Err(Error::PowInvalid) } Ok(sealed) => { @@ -622,14 +623,14 @@ impl MinerService for Miner { } } } else { - info!(target: "miner", "Mined block rejected, PoW hash invalid or out of date."); + info!(target: "miner", "Mined solution rejected: Block unknown or out of date."); Err(Error::PowHashInvalid) }; result.and_then(|sealed| { let n = sealed.header().number(); let h = sealed.header().hash(); try!(chain.import_sealed_block(sealed)); - info!("Mined block imported OK. #{}: {}", n, h.hex()); + info!("Mined block imported OK. #{}: {}", paint(White.bold(), format!("{}", n)), paint(White.bold(), h.hex())); Ok(()) }) } diff --git a/parity/main.rs b/parity/main.rs index 047338bc8..5b728f560 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -184,7 +184,7 @@ fn execute_client(conf: Configuration, spec: Spec, client_config: ClientConfig) let panic_handler = PanicHandler::new_in_arc(); // Setup logging - let logger = setup_log::setup_log(&conf.args.flag_logging); + let logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } @@ -320,6 +320,8 @@ fn execute_export(conf: Configuration) { // Setup panic handler let panic_handler = PanicHandler::new_in_arc(); + // Setup logging + let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } @@ -392,6 +394,8 @@ fn execute_import(conf: Configuration) { // Setup panic handler let panic_handler = PanicHandler::new_in_arc(); + // Setup logging + let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } diff --git a/parity/setup_log.rs b/parity/setup_log.rs index 4ed153fc2..d347a6bf0 100644 --- a/parity/setup_log.rs +++ b/parity/setup_log.rs @@ -19,10 +19,10 @@ use std::env; use std::sync::Arc; use time; use env_logger::LogBuilder; -use util::{RotatingLogger}; +use util::RotatingLogger; /// Sets up the logger -pub fn setup_log(init: &Option) -> Arc { +pub fn setup_log(init: &Option, enable_color: bool) -> Arc { use rlog::*; let mut levels = String::new(); @@ -43,7 +43,7 @@ pub fn setup_log(init: &Option) -> Arc { builder.parse(s); } - let logs = Arc::new(RotatingLogger::new(levels)); + let logs = Arc::new(RotatingLogger::new(levels, enable_color)); let logger = logs.clone(); let format = move |record: &LogRecord| { let timestamp = time::strftime("%Y-%m-%d %H:%M:%S %Z", &time::now()).unwrap(); diff --git a/rpc/src/v1/tests/mocked/ethcore.rs b/rpc/src/v1/tests/mocked/ethcore.rs index 5b88e8756..cbdddc2b0 100644 --- a/rpc/src/v1/tests/mocked/ethcore.rs +++ b/rpc/src/v1/tests/mocked/ethcore.rs @@ -32,7 +32,7 @@ fn client_service() -> Arc { } fn logger() -> Arc { - Arc::new(RotatingLogger::new("rpc=trace".to_owned())) + Arc::new(RotatingLogger::new("rpc=trace".to_owned(), false)) } fn settings() -> Arc { diff --git a/util/Cargo.toml b/util/Cargo.toml index 5cd0a714b..e6f62dd86 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -37,6 +37,7 @@ vergen = "0.1" target_info = "0.1" bigint = { path = "bigint" } chrono = "0.2" +ansi_term = "0.7" [features] default = [] diff --git a/util/src/lib.rs b/util/src/lib.rs index adaf08e77..31e072dc7 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -117,6 +117,7 @@ extern crate libc; extern crate target_info; extern crate bigint; extern crate chrono; +extern crate ansi_term; pub mod standard; #[macro_use] diff --git a/util/src/log.rs b/util/src/log.rs index 172957c13..1dddae1cb 100644 --- a/util/src/log.rs +++ b/util/src/log.rs @@ -20,7 +20,21 @@ use std::env; use rlog::{LogLevelFilter}; use env_logger::LogBuilder; use std::sync::{RwLock, RwLockReadGuard}; +use std::sync::atomic::{Ordering, AtomicBool}; use arrayvec::ArrayVec; +pub use ansi_term::{Colour, Style}; + +lazy_static! { + static ref USE_COLOR: AtomicBool = AtomicBool::new(false); +} + +/// Paint, using colour if desired. +pub fn paint(c: Style, t: String) -> String { + match USE_COLOR.load(Ordering::Relaxed) { + true => format!("{}", c.paint(t)), + false => t, + } +} lazy_static! { static ref LOG_DUMMY: bool = { @@ -57,7 +71,8 @@ impl RotatingLogger { /// Creates new `RotatingLogger` with given levels. /// It does not enforce levels - it's just read only. - pub fn new(levels: String) -> Self { + pub fn new(levels: String, enable_color: bool) -> Self { + USE_COLOR.store(enable_color, Ordering::Relaxed); RotatingLogger { levels: levels, logs: RwLock::new(ArrayVec::<[_; LOG_SIZE]>::new()), @@ -86,7 +101,7 @@ mod test { use super::RotatingLogger; fn logger() -> RotatingLogger { - RotatingLogger::new("test".to_owned()) + RotatingLogger::new("test".to_owned(), false) } #[test] From 93a89049ed10c1f73637dab745d51ca2f16ef01e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 17:50:27 +0200 Subject: [PATCH 21/45] More colour! --- ethcore/src/service.rs | 4 ++-- parity/configuration.rs | 3 ++- parity/main.rs | 6 ++++-- signer/src/authcode_store.rs | 2 +- util/src/network/host.rs | 10 +++++++++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index d8233a4b6..e27dfdee0 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -17,6 +17,7 @@ //! Creates and registers client and network services. use util::*; +use util::Colour::{Yellow, Green}; use util::panics::*; use spec::Spec; use error::*; @@ -71,8 +72,7 @@ impl ClientService { try!(net_service.start()); } - info!("Starting {}", net_service.host_info()); - info!("Configured for {} using {:?} engine", spec.name, spec.engine.name()); + info!("Configured for {} using {} engine", paint(Green.bold(), spec.name.clone()), paint(Yellow.bold(), spec.engine.name().to_owned())); let client = try!(Client::new(config, spec, db_path, miner, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { diff --git a/parity/configuration.rs b/parity/configuration.rs index 0cfb7c44f..0ef2e891b 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -25,6 +25,7 @@ use docopt::Docopt; use die::*; use util::*; +use util::log::Colour::*; use ethcore::account_provider::AccountProvider; use util::network_settings::NetworkSettings; use ethcore::client::{append_path, get_db_path, ClientConfig, DatabaseCompactionProfile, Switch, VMType}; @@ -180,7 +181,7 @@ impl Configuration { let wei_per_usd: f32 = 1.0e18 / usd_per_eth; let gas_per_tx: f32 = 21000.0; let wei_per_gas: f32 = wei_per_usd * usd_per_tx / gas_per_tx; - info!("Using a conversion rate of Ξ1 = US${} ({} wei/gas)", usd_per_eth, wei_per_gas); + info!("Using a conversion rate of Ξ1 = {} ({} wei/gas)", paint(White.bold(), format!("US${}", usd_per_eth)), paint(Yellow.bold(), format!("{}", wei_per_gas))); U256::from_dec_str(&format!("{:.0}", wei_per_gas)).unwrap() } } diff --git a/parity/main.rs b/parity/main.rs index 5b728f560..7809ee07c 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -80,7 +80,7 @@ use std::thread::sleep; use std::time::Duration; use rustc_serialize::hex::FromHex; use ctrlc::CtrlC; -use util::{H256, ToPretty, NetworkConfiguration, PayloadInfo, Bytes, UtilError}; +use util::{H256, ToPretty, NetworkConfiguration, PayloadInfo, Bytes, UtilError, paint, Colour, version}; use util::panics::{MayPanic, ForwardPanic, PanicHandler}; use ethcore::client::{BlockID, BlockChainClient, ClientConfig, get_db_path}; use ethcore::error::{Error, ImportError}; @@ -184,10 +184,12 @@ fn execute_client(conf: Configuration, spec: Spec, client_config: ClientConfig) let panic_handler = PanicHandler::new_in_arc(); // Setup logging - let logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); + let logger = setup_log::setup_log(&conf.args.flag_logging, !conf.args.flag_no_color); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } + info!("Starting {}", paint(Colour::White.bold(), format!("{}", version()))); + let net_settings = conf.net_settings(&spec); let sync_config = conf.sync_config(&spec); diff --git a/signer/src/authcode_store.rs b/signer/src/authcode_store.rs index 92e86a73e..e85633d2c 100644 --- a/signer/src/authcode_store.rs +++ b/signer/src/authcode_store.rs @@ -120,7 +120,7 @@ impl AuthCodes { .filter_map(|f| String::from_utf8(f.to_vec()).ok()) .collect::>() .join("-"); - info!(target: "signer", "New authentication token generated."); + trace!(target: "signer", "New authentication token generated."); self.codes.push(code); Ok(readable_code) } diff --git a/util/src/network/host.rs b/util/src/network/host.rs index 46185482f..a48d1544c 100644 --- a/util/src/network/host.rs +++ b/util/src/network/host.rs @@ -32,6 +32,8 @@ use misc::version; use crypto::*; use sha3::Hashable; use rlp::*; +use log::Colour::White; +use log::paint; use network::session::{Session, SessionData}; use error::*; use io::*; @@ -343,6 +345,7 @@ pub struct Host where Message: Send + Sync + Clone { reserved_nodes: RwLock>, num_sessions: AtomicUsize, stopping: AtomicBool, + first_time: AtomicBool, } impl Host where Message: Send + Sync + Clone { @@ -398,6 +401,7 @@ impl Host where Message: Send + Sync + Clone { reserved_nodes: RwLock::new(HashSet::new()), num_sessions: AtomicUsize::new(0), stopping: AtomicBool::new(false), + first_time: AtomicBool::new(true), }; for n in boot_nodes { @@ -533,7 +537,11 @@ impl Host where Message: Send + Sync + Clone { }; self.info.write().unwrap().public_endpoint = Some(public_endpoint.clone()); - info!("Public node URL: {}", self.external_url().unwrap()); + + if self.first_time.load(AtomicOrdering::Relaxed) { + info!("Public node URL: {}", paint(White.bold(), format!("{}", self.external_url().unwrap()))); + self.first_time.store(false, AtomicOrdering::Relaxed); + } // Initialize discovery. let discovery = { From 6ca2e6b29bd7d86aa9e7887fe06dfca705a3ddb0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 19:19:45 +0200 Subject: [PATCH 22/45] Remove extraneous messages. --- ethcore/src/client/client.rs | 1 - ethcore/src/miner/miner.rs | 18 ++++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 3cc004d20..70205501e 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -863,7 +863,6 @@ impl MiningBlockChainClient for Client { self.miner.update_sealing(self); } - info!("Block {} ({}) submitted and imported.", h.hex(), number); Ok(h) } } diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 8b6e9b49e..d48b7c5dc 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -612,25 +612,19 @@ impl MinerService for Miner { fn submit_seal(&self, chain: &MiningBlockChainClient, pow_hash: H256, seal: Vec) -> Result<(), Error> { let result = if let Some(b) = self.sealing_work.lock().unwrap().take_used_if(|b| &b.hash() == &pow_hash) { - match b.lock().try_seal(self.engine(), seal) { - Err(_) => { - info!(target: "miner", "Mined solution rejected: Invalid."); - Err(Error::PowInvalid) - } - Ok(sealed) => { - info!(target: "miner", "New solution received for #{}: {}", sealed.header().number(), sealed.header().hash()); - Ok(sealed) - } - } + b.lock().try_seal(self.engine(), seal).or_else(|_| { + warn!(target: "miner", "Mined solution rejected: Invalid."); + Err(Error::PowInvalid) + }) } else { - info!(target: "miner", "Mined solution rejected: Block unknown or out of date."); + warn!(target: "miner", "Mined solution rejected: Block unknown or out of date."); Err(Error::PowHashInvalid) }; result.and_then(|sealed| { let n = sealed.header().number(); let h = sealed.header().hash(); try!(chain.import_sealed_block(sealed)); - info!("Mined block imported OK. #{}: {}", paint(White.bold(), format!("{}", n)), paint(White.bold(), h.hex())); + info!(target: "miner", "Mined block imported OK. #{}: {}", paint(White.bold(), format!("{}", n)), paint(White.bold(), h.hex())); Ok(()) }) } From 5a794b21cf5e12f32d721c1eae8fb2dcd58f108a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 29 Jun 2016 20:06:29 +0200 Subject: [PATCH 23/45] Make output less green. --- ethcore/src/service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/src/service.rs b/ethcore/src/service.rs index e27dfdee0..c4cbc497b 100644 --- a/ethcore/src/service.rs +++ b/ethcore/src/service.rs @@ -17,7 +17,7 @@ //! Creates and registers client and network services. use util::*; -use util::Colour::{Yellow, Green}; +use util::Colour::{Yellow, White}; use util::panics::*; use spec::Spec; use error::*; @@ -72,7 +72,7 @@ impl ClientService { try!(net_service.start()); } - info!("Configured for {} using {} engine", paint(Green.bold(), spec.name.clone()), paint(Yellow.bold(), spec.engine.name().to_owned())); + info!("Configured for {} using {} engine", paint(White.bold(), spec.name.clone()), paint(Yellow.bold(), spec.engine.name().to_owned())); let client = try!(Client::new(config, spec, db_path, miner, net_service.io().channel())); panic_handler.forward_from(client.deref()); let client_io = Arc::new(ClientIoHandler { From af65945b58306219fa908cd6ac20f46fb00f1d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 30 Jun 2016 02:24:01 -0400 Subject: [PATCH 24/45] Specifying max open files (#1494) --- util/src/kvdb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/src/kvdb.rs b/util/src/kvdb.rs index aeb8b9fa7..8a19e48bf 100644 --- a/util/src/kvdb.rs +++ b/util/src/kvdb.rs @@ -99,7 +99,7 @@ impl DatabaseConfig { DatabaseConfig { cache_size: Some(cache_size), prefix_size: None, - max_open_files: -1, + max_open_files: 256, compaction: CompactionProfile::default(), } } @@ -122,7 +122,7 @@ impl Default for DatabaseConfig { DatabaseConfig { cache_size: None, prefix_size: None, - max_open_files: -1, + max_open_files: 256, compaction: CompactionProfile::default(), } } From 51c6b85f80a48dcd83420f78e79c48280f9a79a5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 30 Jun 2016 10:07:33 +0200 Subject: [PATCH 25/45] Workaround hyper panic --- ethcore/src/miner/work_notify.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ethcore/src/miner/work_notify.rs b/ethcore/src/miner/work_notify.rs index 6144e2d3d..a153be79f 100644 --- a/ethcore/src/miner/work_notify.rs +++ b/ethcore/src/miner/work_notify.rs @@ -43,9 +43,7 @@ impl WorkPoster { } } }).collect(); - let client = Client::::configure() - .keep_alive(false) - .build().expect("Error creating HTTP client"); + let client = WorkPoster::create_client(); WorkPoster { client: Mutex::new(client), urls: urls, @@ -53,6 +51,13 @@ impl WorkPoster { } } + fn create_client() -> Client { + let client = Client::::configure() + .keep_alive(true) + .build().expect("Error creating HTTP client") as Client; + client + } + pub fn notify(&self, pow_hash: H256, difficulty: U256, number: u64) { // TODO: move this to engine let target = Ethash::difficulty_to_boundary(&difficulty); @@ -60,10 +65,15 @@ impl WorkPoster { let seed_hash = H256::from_slice(&seed_hash[..]); let body = format!(r#"{{ "result": ["0x{}","0x{}","0x{}","0x{:x}"] }}"#, pow_hash.hex(), seed_hash.hex(), target.hex(), number); - let client = self.client.lock().unwrap(); + let mut client = self.client.lock().unwrap(); for u in &self.urls { if let Err(e) = client.request(u.clone(), PostHandler { body: body.clone() }) { - warn!("Error sending HTTP notification to {} : {}", u, e); + warn!("Error sending HTTP notification to {} : {}, retrying", u, e); + // TODO: remove this once https://github.com/hyperium/hyper/issues/848 is fixed + *client = WorkPoster::create_client(); + if let Err(e) = client.request(u.clone(), PostHandler { body: body.clone() }) { + warn!("Error sending HTTP notification to {} : {}", u, e); + } } } } From 5665083e208a88e0fdaeddda41e06f1b36965291 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 12:21:04 +0200 Subject: [PATCH 26/45] UsingQueue can clone rather than just take. --- util/src/using_queue.rs | 44 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/util/src/using_queue.rs b/util/src/using_queue.rs index a5a2b0465..406ed7c06 100644 --- a/util/src/using_queue.rs +++ b/util/src/using_queue.rs @@ -74,6 +74,12 @@ impl UsingQueue where T: Clone { self.in_use.iter().position(|r| predicate(r)).map(|i| self.in_use.remove(i)) } + /// Returns `Some` item which is the first that `f` returns `true` with a reference to it + /// as a parameter or `None` if no such item exists in the queue. + pub fn clone_used_if

(&mut self, predicate: P) -> Option where P: Fn(&T) -> bool { + self.in_use.iter().find(|r| predicate(r)).cloned() + } + /// Returns the most recently pushed block if `f` returns `true` with a reference to it as /// a parameter, otherwise `None`. /// Will not destroy a block if a reference to it has previously been returned by `use_last_ref`, @@ -94,18 +100,52 @@ impl UsingQueue where T: Clone { } #[test] -fn should_find_when_pushed() { +fn should_not_find_when_pushed() { let mut q = UsingQueue::new(2); q.push(1); assert!(q.take_used_if(|i| i == &1).is_none()); } +#[test] +fn should_not_find_when_pushed_with_clone() { + let mut q = UsingQueue::new(2); + q.push(1); + assert!(q.clone_used_if(|i| i == &1).is_none()); +} + #[test] fn should_find_when_pushed_and_used() { let mut q = UsingQueue::new(2); q.push(1); q.use_last_ref(); - assert!(q.take_used_if(|i| i == &1).is_some()); + assert!(q.take_used_if(|i| i == &1).unwrap() == 1); +} + +#[test] +fn should_find_when_pushed_and_used_with_clone() { + let mut q = UsingQueue::new(2); + q.push(1); + q.use_last_ref(); + assert!(q.clone_used_if(|i| i == &1).unwrap() == 1); +} + +#[test] +fn should_not_find_again_when_pushed_and_taken() { + let mut q = UsingQueue::new(2); + q.push(1); + q.use_last_ref(); + assert!(q.take_used_if(|i| i == &1).unwrap() == 1); + assert!(q.clone_used_if(|i| i == &1).is_none()); +} + +#[test] +fn should_find_again_when_pushed_and_cloned() { + let mut q = UsingQueue::new(2); + q.push(1); + q.use_last_ref(); + assert!(q.clone_used_if(|i| i == &1).unwrap() == 1); + assert!(q.clone_used_if(|i| i == &1).unwrap() == 1); + assert!(q.take_used_if(|i| i == &1).unwrap() == 1); } #[test] From 9c07e5c3551523a83ac855adb0a29af0bb4d2aca Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 12:56:58 +0200 Subject: [PATCH 27/45] Optionally clone block behind work-package. --- ethcore/src/client/client.rs | 3 +++ ethcore/src/miner/miner.rs | 9 +++++++-- parity/cli.rs | 5 +++++ parity/configuration.rs | 1 + rpc/src/v1/tests/eth.rs | 3 ++- util/src/using_queue.rs | 30 ++++++++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 3 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 3cc004d20..7cf590173 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -362,6 +362,9 @@ impl Client { let receipts = block.receipts().clone(); let traces = From::from(block.traces().clone().unwrap_or_else(Vec::new)); + // CHECK! I *think* this is fine, even if the state_root is equal to another + // already-imported block of the same number. + // TODO: Prove it with a test. block.drain().commit(number, hash, ancient).expect("State DB commit failed."); // And update the chain after commit to prevent race conditions diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index aea0fd154..25f1adf34 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -64,6 +64,8 @@ pub struct MinerOptions { pub pending_set: PendingSet, /// How many historical work packages can we store before running out? pub work_queue_size: usize, + /// Can we submit two different solutions for the same block and expect both to result in an import? + pub enable_resubmission: bool, } impl Default for MinerOptions { @@ -78,6 +80,7 @@ impl Default for MinerOptions { pending_set: PendingSet::AlwaysQueue, reseal_min_period: Duration::from_secs(0), work_queue_size: 20, + enable_resubmission: true, } } } @@ -251,7 +254,9 @@ impl Miner { let work = { let mut sealing_work = self.sealing_work.lock().unwrap(); - let work = if sealing_work.peek_last_ref().map_or(true, |pb| pb.block().fields().header.hash() != block.block().fields().header.hash()) { + let last_work_hash = sealing_work.peek_last_ref().map(|pb| pb.block().fields().header.hash()); + trace!(target: "miner", "Checking whether we need to reseal: last={:?}, this={:?}", last_work_hash, block.block().fields().header.hash()); + let work = if last_work_hash.map_or(true, |h| h != block.block().fields().header.hash()) { trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); let pow_hash = block.block().fields().header.hash(); let number = block.block().fields().header.number(); @@ -610,7 +615,7 @@ impl MinerService for Miner { } fn submit_seal(&self, chain: &MiningBlockChainClient, pow_hash: H256, seal: Vec) -> Result<(), Error> { - let result = if let Some(b) = self.sealing_work.lock().unwrap().take_used_if(|b| &b.hash() == &pow_hash) { + let result = if let Some(b) = self.sealing_work.lock().unwrap().get_used_if(if self.options.enable_resubmission { GetAction::Clone } else { GetAction::Take }, |b| &b.hash() == &pow_hash) { match b.lock().try_seal(self.engine(), seal) { Err(_) => { info!(target: "miner", "Mined block rejected, PoW was invalid."); diff --git a/parity/cli.rs b/parity/cli.rs index 3c1eee8e8..7ebbcb0aa 100644 --- a/parity/cli.rs +++ b/parity/cli.rs @@ -169,6 +169,10 @@ Sealing/Mining Options: more than 32 characters. --tx-queue-size LIMIT Maximum amount of transactions in the queue (waiting to be included in next block) [default: 1024]. + --remove-solved Move solved blocks from the work package queue + instead of cloning them. This gives a slightly + faster import speed, but means that extra solutions + submitted for the same work package will go unused. --notify-work URLS URLs to which work package notifications are pushed. URLS should be a comma-delimited list of HTTP URLs. @@ -313,6 +317,7 @@ pub struct Args { pub flag_reseal_on_txs: String, pub flag_reseal_min_period: u64, pub flag_work_queue_size: usize, + pub flag_remove_solved: bool, pub flag_tx_gas_limit: Option, pub flag_relay_set: String, pub flag_author: Option, diff --git a/parity/configuration.rs b/parity/configuration.rs index 0cfb7c44f..29a556544 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -111,6 +111,7 @@ impl Configuration { }, reseal_min_period: Duration::from_millis(self.args.flag_reseal_min_period), work_queue_size: self.args.flag_work_queue_size, + enable_resubmission: !self.args.flag_remove_solved, } } diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 4eb80b1c0..2965a62d2 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -52,6 +52,7 @@ fn sync_provider() -> Arc { fn miner_service(spec: Spec, accounts: Arc) -> Arc { Miner::new( MinerOptions { + new_work_notify: vec![], force_sealing: true, reseal_on_external_tx: true, reseal_on_own_tx: true, @@ -60,7 +61,7 @@ fn miner_service(spec: Spec, accounts: Arc) -> Arc { pending_set: PendingSet::SealingOrElseQueue, reseal_min_period: Duration::from_secs(0), work_queue_size: 50, - new_work_notify: vec![], + enable_resubmission: true, }, spec, Some(accounts) diff --git a/util/src/using_queue.rs b/util/src/using_queue.rs index 406ed7c06..e5e1a5a58 100644 --- a/util/src/using_queue.rs +++ b/util/src/using_queue.rs @@ -27,6 +27,14 @@ pub struct UsingQueue where T: Clone { max_size: usize, } +/// Take an item or just clone it? +pub enum GetAction { + /// Remove the item, faster but you can't get it back. + Take, + /// Clone the item, slower but you can get it again. + Clone, +} + impl UsingQueue where T: Clone { /// Create a new struct with a maximum size of `max_size`. pub fn new(max_size: usize) -> UsingQueue { @@ -80,6 +88,14 @@ impl UsingQueue where T: Clone { self.in_use.iter().find(|r| predicate(r)).cloned() } + /// Fork-function for `take_used_if` and `clone_used_if`. + pub fn get_used_if

(&mut self, action: GetAction, predicate: P) -> Option where P: Fn(&T) -> bool { + match action { + GetAction::Take => self.take_used_if(predicate), + GetAction::Clone => self.clone_used_if(predicate), + } + } + /// Returns the most recently pushed block if `f` returns `true` with a reference to it as /// a parameter, otherwise `None`. /// Will not destroy a block if a reference to it has previously been returned by `use_last_ref`, @@ -121,6 +137,20 @@ fn should_find_when_pushed_and_used() { assert!(q.take_used_if(|i| i == &1).unwrap() == 1); } +#[test] +fn should_have_same_semantics_for_get_take_clone() { + let mut q = UsingQueue::new(2); + q.push(1); + assert!(q.get_used_if(GetAction::Clone, |i| i == &1).is_none()); + assert!(q.get_used_if(GetAction::Take, |i| i == &1).is_none()); + q.use_last_ref(); + assert!(q.get_used_if(GetAction::Clone, |i| i == &1).unwrap() == 1); + assert!(q.get_used_if(GetAction::Clone, |i| i == &1).unwrap() == 1); + assert!(q.get_used_if(GetAction::Take, |i| i == &1).unwrap() == 1); + assert!(q.get_used_if(GetAction::Clone, |i| i == &1).is_none()); + assert!(q.get_used_if(GetAction::Take, |i| i == &1).is_none()); +} + #[test] fn should_find_when_pushed_and_used_with_clone() { let mut q = UsingQueue::new(2); From dff7d9603cf093864fffda700d8b0994991e56a9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 13:12:15 +0200 Subject: [PATCH 28/45] Fix for fake new work packages. --- ethcore/src/miner/miner.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 25f1adf34..4672ef3fa 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -157,9 +157,10 @@ impl Miner { fn prepare_sealing(&self, chain: &MiningBlockChainClient) { trace!(target: "miner", "prepare_sealing: entering"); - let (transactions, mut open_block) = { + let (transactions, mut open_block, last_work_hash) = { let transactions = {self.transaction_queue.lock().unwrap().top_transactions()}; let mut sealing_work = self.sealing_work.lock().unwrap(); + let last_work_hash = sealing_work.peek_last_ref().map(|pb| pb.block().fields().header.hash()); let best_hash = chain.best_block_header().sha3(); /* // check to see if last ClosedBlock in would_seals is actually same parent block. @@ -186,7 +187,7 @@ impl Miner { ) } }; - (transactions, open_block) + (transactions, open_block, last_work_hash) }; let mut invalid_transactions = HashSet::new(); @@ -254,7 +255,6 @@ impl Miner { let work = { let mut sealing_work = self.sealing_work.lock().unwrap(); - let last_work_hash = sealing_work.peek_last_ref().map(|pb| pb.block().fields().header.hash()); trace!(target: "miner", "Checking whether we need to reseal: last={:?}, this={:?}", last_work_hash, block.block().fields().header.hash()); let work = if last_work_hash.map_or(true, |h| h != block.block().fields().header.hash()) { trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); From 6ae467252c3cd8811a66bf54ebc5ea5a743317b8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 15:49:00 +0200 Subject: [PATCH 29/45] Fix no colour on windows. (#1498) --- parity/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parity/main.rs b/parity/main.rs index 7809ee07c..d466987ef 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -184,7 +184,7 @@ fn execute_client(conf: Configuration, spec: Spec, client_config: ClientConfig) let panic_handler = PanicHandler::new_in_arc(); // Setup logging - let logger = setup_log::setup_log(&conf.args.flag_logging, !conf.args.flag_no_color); + let logger = setup_log::setup_log(&conf.args.flag_logging, conf.have_color()); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } @@ -323,7 +323,7 @@ fn execute_export(conf: Configuration) { let panic_handler = PanicHandler::new_in_arc(); // Setup logging - let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); + let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.have_color()); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } @@ -397,7 +397,7 @@ fn execute_import(conf: Configuration) { let panic_handler = PanicHandler::new_in_arc(); // Setup logging - let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.args.flag_no_color); + let _logger = setup_log::setup_log(&conf.args.flag_logging, conf.have_color()); // Raise fdlimit unsafe { ::fdlimit::raise_fd_limit(); } From 6616b5e17d2810e9dffa3f021106dbf1e409e1ad Mon Sep 17 00:00:00 2001 From: goldylucks Date: Thu, 30 Jun 2016 18:14:20 +0200 Subject: [PATCH 30/45] bump status page v0.5.1 --- Cargo.lock | 6 +++--- dapps/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18168f795..6faa0cf32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,7 +286,7 @@ dependencies = [ "mime_guess 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-dapps 0.3.0 (git+https://github.com/ethcore/parity-dapps-rs.git)", "parity-dapps-builtins 0.5.2 (git+https://github.com/ethcore/parity-dapps-builtins-rs.git)", - "parity-dapps-status 0.5.0 (git+https://github.com/ethcore/parity-dapps-status-rs.git)", + "parity-dapps-status 0.5.1 (git+https://github.com/ethcore/parity-dapps-status-rs.git)", "parity-dapps-wallet 0.6.1 (git+https://github.com/ethcore/parity-dapps-wallet-rs.git)", "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -919,8 +919,8 @@ dependencies = [ [[package]] name = "parity-dapps-status" -version = "0.5.0" -source = "git+https://github.com/ethcore/parity-dapps-status-rs.git#0cdd3512004e403aff7da3b8c16ba0bf5d6c911c" +version = "0.5.1" +source = "git+https://github.com/ethcore/parity-dapps-status-rs.git#110ef2e66142ec8dc15fc40b8ddda5ed3bcfc1fb" dependencies = [ "parity-dapps 0.3.0 (git+https://github.com/ethcore/parity-dapps-rs.git)", ] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 6aaceb50a..d33bd6702 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -22,7 +22,7 @@ ethcore-rpc = { path = "../rpc" } ethcore-util = { path = "../util" } parity-dapps = { git = "https://github.com/ethcore/parity-dapps-rs.git", version = "0.3" } # List of apps -parity-dapps-status = { git = "https://github.com/ethcore/parity-dapps-status-rs.git", version = "0.5.0" } +parity-dapps-status = { git = "https://github.com/ethcore/parity-dapps-status-rs.git", version = "0.5.1" } parity-dapps-builtins = { git = "https://github.com/ethcore/parity-dapps-builtins-rs.git", version = "0.5.2" } parity-dapps-wallet = { git = "https://github.com/ethcore/parity-dapps-wallet-rs.git", version = "0.6.0", optional = true } parity-dapps-dao = { git = "https://github.com/ethcore/parity-dapps-dao-rs.git", version = "0.4.0", optional = true } From 673c5afd4ddb3f3af050e447087c5b58f632d823 Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Thu, 30 Jun 2016 18:56:16 +0200 Subject: [PATCH 31/45] Update install-parity.sh --- install-parity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-parity.sh b/install-parity.sh index eadf698db..eae35f764 100755 --- a/install-parity.sh +++ b/install-parity.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -PARITY_DEB_URL=https://github.com/ethcore/parity/releases/download/v1.2.0/parity_linux_1.2.0-0_amd64.deb +PARITY_DEB_URL=https://github.com/ethcore/parity/releases/download/v1.2.1/parity_linux_1.2.1-0_amd64.deb function run_installer() From d3cb79833b4a087f7f258a478cc6501521103a00 Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:07:22 +0700 Subject: [PATCH 32/45] Update Ubuntu Dockerfile [ci skip] --- docker/ubuntu/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile index 9999909c3..6c80238a1 100644 --- a/docker/ubuntu/Dockerfile +++ b/docker/ubuntu/Dockerfile @@ -27,7 +27,9 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ + git checkout master && \ + git pull && \ cargo build --release --verbose && \ - ls /build/parity/target/release/parity && \ + ls /build/parity/target/release/parity && \ strip /build/parity/target/release/parity RUN file /build/parity/target/release/parity From 8134a89b87f398e27094eae6c810c38da4e2f1da Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:08:07 +0700 Subject: [PATCH 33/45] Update Ubuntu-jit Dockerfile [ci skip] --- docker/ubuntu-jit/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/ubuntu-jit/Dockerfile b/docker/ubuntu-jit/Dockerfile index 89c38cee7..07f351460 100644 --- a/docker/ubuntu-jit/Dockerfile +++ b/docker/ubuntu-jit/Dockerfile @@ -47,6 +47,8 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ + git checkout master && \ + git pull && \ cargo build --release --features ethcore/jit --verbose && \ ls /build/parity/target/release/parity && \ strip /build/parity/target/release/parity From bb70cd889eccef2070264b09969e339ced34dcfa Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:08:58 +0700 Subject: [PATCH 34/45] Update Ubuntu-arm Dockerfile [ci skip] --- docker/ubuntu-arm/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/ubuntu-arm/Dockerfile b/docker/ubuntu-arm/Dockerfile index fab325e5e..075f7324e 100644 --- a/docker/ubuntu-arm/Dockerfile +++ b/docker/ubuntu-arm/Dockerfile @@ -36,6 +36,8 @@ ENV CC arm-linux-gnueabihf-gcc # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ + git checkout master && \ + git pull && \ mkdir -p .cargo && \ echo '[target.armv7-unknown-linux-gnueabihf]\n\ linker = "arm-linux-gnueabihf-gcc"\n'\ From bee2102424a52c690f41aea0da071c3e4d9e31a7 Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:09:48 +0700 Subject: [PATCH 35/45] Update Dockerfile ubuntu-aarch64 [ci skip] --- docker/ubuntu-aarch64/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/ubuntu-aarch64/Dockerfile b/docker/ubuntu-aarch64/Dockerfile index 3212f84d0..c9939074c 100644 --- a/docker/ubuntu-aarch64/Dockerfile +++ b/docker/ubuntu-aarch64/Dockerfile @@ -36,6 +36,8 @@ ENV CC aarch64-linux-gnu-gcc # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ + git checkout master && \ + git pull && \ mkdir -p .cargo && \ echo '[target.aarch64-unknown-linux-gnu]\n\ linker = "aarch64-linux-gnu-gcc"\n'\ From 90a4475f526537ddecc085e9ee1155047ce4abeb Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:10:58 +0700 Subject: [PATCH 36/45] Update CentOS Dockerfile [ci skip] --- docker/centos/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile index 56015422c..ea0571cca 100644 --- a/docker/centos/Dockerfile +++ b/docker/centos/Dockerfile @@ -20,6 +20,8 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity&&\ + git checkout master && \ + git pull && \ ls -a&&\ cargo build --release --verbose && \ ls /build/parity/target/release/parity && \ From f370e17a544c9fd7af117611036cb9b0b7a25d1d Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:26:56 +0700 Subject: [PATCH 37/45] Update CentOS Dockerfile [ci skip] --- docker/centos/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile index ea0571cca..9fd33f4ff 100644 --- a/docker/centos/Dockerfile +++ b/docker/centos/Dockerfile @@ -20,7 +20,7 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity&&\ - git checkout master && \ + git checkout beta && \ git pull && \ ls -a&&\ cargo build --release --verbose && \ From 6bdbc2cb9344614dedacbaf821ca451b259ae3c1 Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:27:36 +0700 Subject: [PATCH 38/45] Update Dockerfile ubuntu-aarch64 [ci skip] --- docker/ubuntu-aarch64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ubuntu-aarch64/Dockerfile b/docker/ubuntu-aarch64/Dockerfile index c9939074c..38ed4c9b1 100644 --- a/docker/ubuntu-aarch64/Dockerfile +++ b/docker/ubuntu-aarch64/Dockerfile @@ -36,7 +36,7 @@ ENV CC aarch64-linux-gnu-gcc # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ - git checkout master && \ + git checkout beta && \ git pull && \ mkdir -p .cargo && \ echo '[target.aarch64-unknown-linux-gnu]\n\ From 6bbd237711f8fa4d1214de68ba1bfd4d133ec55d Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:28:18 +0700 Subject: [PATCH 39/45] Update Ubuntu-arm Dockerfile [ci skip] --- docker/ubuntu-arm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ubuntu-arm/Dockerfile b/docker/ubuntu-arm/Dockerfile index 075f7324e..b59573a0f 100644 --- a/docker/ubuntu-arm/Dockerfile +++ b/docker/ubuntu-arm/Dockerfile @@ -36,7 +36,7 @@ ENV CC arm-linux-gnueabihf-gcc # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ - git checkout master && \ + git checkout beta && \ git pull && \ mkdir -p .cargo && \ echo '[target.armv7-unknown-linux-gnueabihf]\n\ From c8c7fbbf277555d9bfe35dede9305449f115bebe Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:29:00 +0700 Subject: [PATCH 40/45] Update Ubuntu-jit Dockerfile [ci skip] --- docker/ubuntu-jit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ubuntu-jit/Dockerfile b/docker/ubuntu-jit/Dockerfile index 07f351460..666b16904 100644 --- a/docker/ubuntu-jit/Dockerfile +++ b/docker/ubuntu-jit/Dockerfile @@ -47,7 +47,7 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ - git checkout master && \ + git checkout beta && \ git pull && \ cargo build --release --features ethcore/jit --verbose && \ ls /build/parity/target/release/parity && \ From 038ef2b3c30c2d7f8cb57ea78fbddbb223562510 Mon Sep 17 00:00:00 2001 From: "Denis S. Soldatov aka General-Beck" Date: Fri, 1 Jul 2016 03:29:45 +0700 Subject: [PATCH 41/45] Update Ubuntu Dockerfile [ci skip] --- docker/ubuntu/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile index 6c80238a1..4c82e1ecc 100644 --- a/docker/ubuntu/Dockerfile +++ b/docker/ubuntu/Dockerfile @@ -27,7 +27,7 @@ g++ -v # build parity RUN git clone https://github.com/ethcore/parity && \ cd parity && \ - git checkout master && \ + git checkout beta && \ git pull && \ cargo build --release --verbose && \ ls /build/parity/target/release/parity && \ From 29b6ba4a87579423019cdcaa76e842a9a2097170 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 22:35:59 +0200 Subject: [PATCH 42/45] Fix the reseal mechanism. --- ethcore/src/miner/miner.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 7b101afc7..35c9998c7 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -158,7 +158,7 @@ impl Miner { fn prepare_sealing(&self, chain: &MiningBlockChainClient) { trace!(target: "miner", "prepare_sealing: entering"); - let (transactions, mut open_block, last_work_hash) = { + let (transactions, mut open_block, original_work_hash) = { let transactions = {self.transaction_queue.lock().unwrap().top_transactions()}; let mut sealing_work = self.sealing_work.lock().unwrap(); let last_work_hash = sealing_work.peek_last_ref().map(|pb| pb.block().fields().header.hash()); @@ -254,23 +254,27 @@ impl Miner { } } - let work = { + let (work, is_new) = { let mut sealing_work = self.sealing_work.lock().unwrap(); - trace!(target: "miner", "Checking whether we need to reseal: last={:?}, this={:?}", last_work_hash, block.block().fields().header.hash()); - let work = if last_work_hash.map_or(true, |h| h != block.block().fields().header.hash()) { + let last_work_hash = sealing_work.peek_last_ref().map(|pb| pb.block().fields().header.hash()); + trace!(target: "miner", "Checking whether we need to reseal: orig={:?} last={:?}, this={:?}", original_work_hash, last_work_hash, block.block().fields().header.hash()); + let (work, is_new) = if last_work_hash.map_or(true, |h| h != block.block().fields().header.hash()) { trace!(target: "miner", "Pushing a new, refreshed or borrowed pending {}...", block.block().fields().header.hash()); let pow_hash = block.block().fields().header.hash(); let number = block.block().fields().header.number(); let difficulty = *block.block().fields().header.difficulty(); + let is_new = original_work_hash.map_or(false, |h| block.block().fields().header.hash() == h); sealing_work.push(block); - Some((pow_hash, difficulty, number)) + (Some((pow_hash, difficulty, number)), is_new) } else { - None + (None, false) }; trace!(target: "miner", "prepare_sealing: leaving (last={:?})", sealing_work.peek_last_ref().map(|b| b.block().fields().header.hash())); - work + (work, is_new) }; - work.map(|(pow_hash, difficulty, number)| self.work_poster.as_ref().map(|ref p| p.notify(pow_hash, difficulty, number))); + if is_new { + work.map(|(pow_hash, difficulty, number)| self.work_poster.as_ref().map(|ref p| p.notify(pow_hash, difficulty, number))); + } } fn update_gas_limit(&self, chain: &MiningBlockChainClient) { From 1d780f456ac7d7bf09af7df366478289788d4d72 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Jun 2016 23:14:54 +0200 Subject: [PATCH 43/45] Fix is_new. --- ethcore/src/miner/miner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 35c9998c7..d3e32c797 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -263,7 +263,7 @@ impl Miner { let pow_hash = block.block().fields().header.hash(); let number = block.block().fields().header.number(); let difficulty = *block.block().fields().header.difficulty(); - let is_new = original_work_hash.map_or(false, |h| block.block().fields().header.hash() == h); + let is_new = original_work_hash.map_or(true, |h| block.block().fields().header.hash() != h); sealing_work.push(block); (Some((pow_hash, difficulty, number)), is_new) } else { From d42ea6b69e29c4c04af146fff46ef00206c98a6d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 1 Jul 2016 14:32:59 +0200 Subject: [PATCH 44/45] modify account_db to work on address hash, not address --- ethcore/src/account_db.rs | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/ethcore/src/account_db.rs b/ethcore/src/account_db.rs index 7337940da..df7666b27 100644 --- a/ethcore/src/account_db.rs +++ b/ethcore/src/account_db.rs @@ -3,24 +3,30 @@ use util::*; static NULL_RLP_STATIC: [u8; 1] = [0x80; 1]; +#[inline] +fn combine_key<'a>(address_hash: &'a H256, key: &'a H256) -> H256 { + address_hash ^ key +} + // TODO: introduce HashDBMut? /// DB backend wrapper for Account trie /// Transforms trie node keys for the database pub struct AccountDB<'db> { db: &'db HashDB, - address: H256, -} - -#[inline] -fn combine_key<'a>(address: &'a H256, key: &'a H256) -> H256 { - address ^ key + address_hash: H256, } impl<'db> AccountDB<'db> { - pub fn new(db: &'db HashDB, address: &Address) -> AccountDB<'db> { + /// Create a new AccountDB from an address. + pub fn new(db: &'db HashDB, address: &Address) -> Self { + Self::from_hash(db, address.sha3()) + } + + /// Create a new AcountDB from an address' hash. + pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self { AccountDB { db: db, - address: address.into(), + address_hash: address_hash, } } } @@ -34,14 +40,14 @@ impl<'db> HashDB for AccountDB<'db>{ if key == &SHA3_NULL_RLP { return Some(&NULL_RLP_STATIC); } - self.db.get(&combine_key(&self.address, key)) + self.db.get(&combine_key(&self.address_hash, key)) } fn contains(&self, key: &H256) -> bool { if key == &SHA3_NULL_RLP { return true; } - self.db.contains(&combine_key(&self.address, key)) + self.db.contains(&combine_key(&self.address_hash, key)) } fn insert(&mut self, _value: &[u8]) -> H256 { @@ -60,20 +66,26 @@ impl<'db> HashDB for AccountDB<'db>{ /// DB backend wrapper for Account trie pub struct AccountDBMut<'db> { db: &'db mut HashDB, - address: H256, + address_hash: H256, } impl<'db> AccountDBMut<'db> { - pub fn new(db: &'db mut HashDB, address: &Address) -> AccountDBMut<'db> { + /// Create a new AccountDB from an address. + pub fn new(db: &'db mut HashDB, address: &Address) -> Self { + Self::from_hash(db, address.sha3()) + } + + /// Create a new AcountDB from an address' hash. + pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self { AccountDBMut { db: db, - address: address.into(), + address_hash: address_hash, } } #[allow(dead_code)] pub fn immutable(&'db self) -> AccountDB<'db> { - AccountDB { db: self.db, address: self.address.clone() } + AccountDB { db: self.db, address_hash: self.address_hash.clone() } } } @@ -86,14 +98,14 @@ impl<'db> HashDB for AccountDBMut<'db>{ if key == &SHA3_NULL_RLP { return Some(&NULL_RLP_STATIC); } - self.db.get(&combine_key(&self.address, key)) + self.db.get(&combine_key(&self.address_hash, key)) } fn contains(&self, key: &H256) -> bool { if key == &SHA3_NULL_RLP { return true; } - self.db.contains(&combine_key(&self.address, key)) + self.db.contains(&combine_key(&self.address_hash, key)) } fn insert(&mut self, value: &[u8]) -> H256 { @@ -101,7 +113,7 @@ impl<'db> HashDB for AccountDBMut<'db>{ return SHA3_NULL_RLP.clone(); } let k = value.sha3(); - let ak = combine_key(&self.address, &k); + let ak = combine_key(&self.address_hash, &k); self.db.emplace(ak, value.to_vec()); k } @@ -110,7 +122,7 @@ impl<'db> HashDB for AccountDBMut<'db>{ if key == SHA3_NULL_RLP { return; } - let key = combine_key(&self.address, &key); + let key = combine_key(&self.address_hash, &key); self.db.emplace(key, value.to_vec()) } @@ -118,7 +130,7 @@ impl<'db> HashDB for AccountDBMut<'db>{ if key == &SHA3_NULL_RLP { return; } - let key = combine_key(&self.address, key); + let key = combine_key(&self.address_hash, key); self.db.remove(&key) } } From 601ebcf3cc7a97bd51c7959621443f3d2ff29b50 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 4 Jul 2016 12:47:03 +0200 Subject: [PATCH 45/45] add a database migration for new accountdb --- ethcore/src/migrations/extras/mod.rs | 2 +- ethcore/src/migrations/mod.rs | 1 + ethcore/src/migrations/state/mod.rs | 5 +++ ethcore/src/migrations/state/v7.rs | 31 +++++++++++++++++ parity/migration.rs | 51 ++++++++++++++++++---------- 5 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 ethcore/src/migrations/state/mod.rs create mode 100644 ethcore/src/migrations/state/v7.rs diff --git a/ethcore/src/migrations/extras/mod.rs b/ethcore/src/migrations/extras/mod.rs index c4d4790dc..28bbb2856 100644 --- a/ethcore/src/migrations/extras/mod.rs +++ b/ethcore/src/migrations/extras/mod.rs @@ -2,4 +2,4 @@ mod v6; -pub use self::v6::ToV6; +pub use self::v6::ToV6; \ No newline at end of file diff --git a/ethcore/src/migrations/mod.rs b/ethcore/src/migrations/mod.rs index 1473ced9c..6d86a122f 100644 --- a/ethcore/src/migrations/mod.rs +++ b/ethcore/src/migrations/mod.rs @@ -1,3 +1,4 @@ //! Database migrations. pub mod extras; +pub mod state; diff --git a/ethcore/src/migrations/state/mod.rs b/ethcore/src/migrations/state/mod.rs new file mode 100644 index 000000000..88d4369a2 --- /dev/null +++ b/ethcore/src/migrations/state/mod.rs @@ -0,0 +1,5 @@ +//! State database migrations. + +mod v7; + +pub use self::v7::ToV7; \ No newline at end of file diff --git a/ethcore/src/migrations/state/v7.rs b/ethcore/src/migrations/state/v7.rs new file mode 100644 index 000000000..aadf9ac62 --- /dev/null +++ b/ethcore/src/migrations/state/v7.rs @@ -0,0 +1,31 @@ +use util::hash::{FixedHash, H256}; +use util::migration::Migration; +use util::sha3::Hashable; + +/// This migration migrates the state db to use an accountdb which ensures uniqueness +/// using an address' hash as opposed to the address itself. +pub struct ToV7; + +impl Migration for ToV7 { + fn version(&self) -> u32 { + 7 + } + + fn simple_migrate(&self, mut key: Vec, value: Vec) -> Option<(Vec, Vec)> { + let val_hash = value.sha3(); + assert!(key.len() == 32); // all keys in the state db are hashes. + let key_h = H256::from_slice(&key[..]); + if key_h != val_hash { + // this is a key which has been xor'd with an address. + // recover the address + let address = key_h ^ val_hash; + let address_hash = address.sha3(); + + let new_key = address_hash ^ val_hash; + key.copy_from_slice(&new_key[..]); + } + // nothing to do here + Some((key, value)) + } +} + diff --git a/parity/migration.rs b/parity/migration.rs index df2c2116f..af09709a9 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -17,7 +17,7 @@ use std::fs; use std::fs::File; use std::io::{Read, Write, Error as IoError, ErrorKind}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::fmt::{Display, Formatter, Error as FmtError}; use util::migration::{Manager as MigrationManager, Config as MigrationConfig, MigrationIterator}; use util::kvdb::{Database, DatabaseConfig, CompactionProfile}; @@ -26,7 +26,7 @@ use ethcore::migrations; /// Database is assumed to be at default version, when no version file is found. const DEFAULT_VERSION: u32 = 5; /// Current version of database models. -const CURRENT_VERSION: u32 = 6; +const CURRENT_VERSION: u32 = 7; /// Defines how many items are migrated to the new version of database at once. const BATCH_SIZE: usize = 1024; /// Version file name. @@ -66,15 +66,15 @@ impl From for Error { } /// Returns the version file path. -fn version_file_path(path: &PathBuf) -> PathBuf { - let mut file_path = path.clone(); +fn version_file_path(path: &Path) -> PathBuf { + let mut file_path = path.to_owned(); file_path.push(VERSION_FILE_NAME); file_path } /// Reads current database version from the file at given path. /// If the file does not exist returns `DEFAULT_VERSION`. -fn current_version(path: &PathBuf) -> Result { +fn current_version(path: &Path) -> Result { match File::open(version_file_path(path)) { Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(DEFAULT_VERSION), Err(_) => Err(Error::UnknownDatabaseVersion), @@ -88,7 +88,7 @@ fn current_version(path: &PathBuf) -> Result { /// Writes current database version to the file. /// Creates a new file if the version file does not exist yet. -fn update_version(path: &PathBuf) -> Result<(), Error> { +fn update_version(path: &Path) -> Result<(), Error> { try!(fs::create_dir_all(path)); let mut file = try!(File::create(version_file_path(path))); try!(file.write_all(format!("{}", CURRENT_VERSION).as_bytes())); @@ -96,30 +96,37 @@ fn update_version(path: &PathBuf) -> Result<(), Error> { } /// Blocks database path. -fn blocks_database_path(path: &PathBuf) -> PathBuf { - let mut blocks_path = path.clone(); +fn blocks_database_path(path: &Path) -> PathBuf { + let mut blocks_path = path.to_owned(); blocks_path.push("blocks"); blocks_path } /// Extras database path. -fn extras_database_path(path: &PathBuf) -> PathBuf { - let mut extras_path = path.clone(); +fn extras_database_path(path: &Path) -> PathBuf { + let mut extras_path = path.to_owned(); extras_path.push("extras"); extras_path } +/// Extras database path. +fn state_database_path(path: &Path) -> PathBuf { + let mut state_path = path.to_owned(); + state_path.push("state"); + state_path +} + /// Temporary database path used for migration. -fn temp_database_path(path: &PathBuf) -> PathBuf { - let mut temp_path = path.clone(); +fn temp_database_path(path: &Path) -> PathBuf { + let mut temp_path = path.to_owned(); temp_path.pop(); temp_path.push("temp_migration"); temp_path } /// Database backup -fn backup_database_path(path: &PathBuf) -> PathBuf { - let mut backup_path = path.clone(); +fn backup_database_path(path: &Path) -> PathBuf { + let mut backup_path = path.to_owned(); backup_path.pop(); backup_path.push("temp_backup"); backup_path @@ -132,19 +139,26 @@ fn default_migration_settings() -> MigrationConfig { } } -/// Migrations on blocks database. +/// Migrations on the blocks database. fn blocks_database_migrations() -> Result { let manager = MigrationManager::new(default_migration_settings()); Ok(manager) } -/// Migrations on extras database. +/// Migrations on the extras database. fn extras_database_migrations() -> Result { let mut manager = MigrationManager::new(default_migration_settings()); try!(manager.add_migration(migrations::extras::ToV6).map_err(|_| Error::MigrationImpossible)); Ok(manager) } +/// Migrations on the state database. +fn state_database_migrations() -> Result { + let mut manager = MigrationManager::new(default_migration_settings()); + try!(manager.add_migration(migrations::state::ToV7).map_err(|_| Error::MigrationImpossible)); + Ok(manager) +} + /// Migrates database at given position with given migration rules. fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) -> Result<(), Error> { // check if migration is needed @@ -192,12 +206,12 @@ fn migrate_database(version: u32, path: PathBuf, migrations: MigrationManager) - Ok(()) } -fn exists(path: &PathBuf) -> bool { +fn exists(path: &Path) -> bool { fs::metadata(path).is_ok() } /// Migrates the database. -pub fn migrate(path: &PathBuf) -> Result<(), Error> { +pub fn migrate(path: &Path) -> Result<(), Error> { // read version file. let version = try!(current_version(path)); @@ -207,6 +221,7 @@ pub fn migrate(path: &PathBuf) -> Result<(), Error> { println!("Migrating database from version {} to {}", version, CURRENT_VERSION); try!(migrate_database(version, blocks_database_path(path), try!(blocks_database_migrations()))); try!(migrate_database(version, extras_database_path(path), try!(extras_database_migrations()))); + try!(migrate_database(version, state_database_path(path), try!(state_database_migrations()))); println!("Migration finished"); }