From 756f96413045e63a3dccf801f992c46ed22f7dfb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 11 Mar 2016 12:54:48 +0100 Subject: [PATCH] JournalDB -> Box, and it's a trait. --- ethcore/src/block.rs | 16 ++--- ethcore/src/client.rs | 12 ++-- ethcore/src/state.rs | 26 +++---- ethcore/src/tests/helpers.rs | 8 +-- util/src/hashdb.rs | 15 ++++- util/src/journaldb.rs | 127 +++++++++++++++++++---------------- util/src/overlaydb.rs | 1 + 7 files changed, 116 insertions(+), 89 deletions(-) diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index b3894db94..ea9e91781 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -171,7 +171,7 @@ pub struct SealedBlock { impl<'x> OpenBlock<'x> { /// Create a new OpenBlock ready for transaction pushing. - pub fn new(engine: &'x Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { + pub fn new(engine: &'x Engine, db: Box, parent: &Header, last_hashes: LastHashes, author: Address, extra_data: Bytes) -> Self { let mut r = OpenBlock { block: ExecutedBlock::new(State::from_existing(db, parent.state_root().clone(), engine.account_start_nonce())), engine: engine, @@ -317,7 +317,7 @@ impl ClosedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> JournalDB { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl SealedBlock { @@ -331,7 +331,7 @@ impl SealedBlock { } /// Drop this object and return the underlieing database. - pub fn drain(self) -> JournalDB { self.block.state.drop().1 } + pub fn drain(self) -> Box { self.block.state.drop().1 } } impl IsBlock for SealedBlock { @@ -339,10 +339,10 @@ impl IsBlock for SealedBlock { } /// Enact the block given by block header, transactions and uncles -pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Header], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { { if ::log::max_log_level() >= ::log::LogLevel::Trace { - let s = State::from_existing(db.clone(), parent.state_root().clone(), engine.account_start_nonce()); + let s = State::from_existing(db.spawn(), parent.state_root().clone(), engine.account_start_nonce()); trace!("enact(): root={}, author={}, author_balance={}\n", s.root(), header.author(), s.balance(&header.author())); } } @@ -357,20 +357,20 @@ pub fn enact(header: &Header, transactions: &[SignedTransaction], uncles: &[Head } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_bytes(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let block = BlockView::new(block_bytes); let header = block.header(); enact(&header, &block.transactions(), &block.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header -pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_verified(block: &PreverifiedBlock, engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let view = BlockView::new(&block.bytes); enact(&block.header, &block.transactions, &view.uncles(), engine, db, parent, last_hashes) } /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header. Seal the block aferwards -pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: JournalDB, parent: &Header, last_hashes: LastHashes) -> Result { +pub fn enact_and_seal(block_bytes: &[u8], engine: &Engine, db: Box, parent: &Header, last_hashes: LastHashes) -> Result { let header = BlockView::new(block_bytes).header_view(); Ok(try!(try!(enact_bytes(block_bytes, engine, db, parent, last_hashes)).seal(engine, header.seal()))) } diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index b342cef15..7ccf094d2 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -214,7 +214,7 @@ impl ClientReport { pub struct Client where V: Verifier { chain: Arc>, engine: Arc>, - state_db: Mutex, + state_db: Mutex>, block_queue: RwLock, report: RwLock, import_lock: Mutex<()>, @@ -253,8 +253,8 @@ impl Client where V: Verifier { state_path.push("state"); let engine = Arc::new(try!(spec.to_engine())); - let mut state_db = JournalDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal); - if state_db.is_empty() && engine.spec().ensure_db_good(&mut state_db) { + let mut state_db = Box::new(OptionOneDB::from_prefs(state_path.to_str().unwrap(), config.prefer_journal)); + if state_db.is_empty() && engine.spec().ensure_db_good(state_db.deref_mut()) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } @@ -336,7 +336,7 @@ impl Client where V: Verifier { // Enact Verified Block let parent = chain_has_parent.unwrap(); let last_hashes = self.build_last_hashes(header.parent_hash.clone()); - let db = self.state_db.lock().unwrap().clone(); + let db = self.state_db.lock().unwrap().spawn(); let enact_result = enact_verified(&block, engine, db, &parent, last_hashes); if let Err(e) = enact_result { @@ -438,7 +438,7 @@ impl Client where V: Verifier { /// Get a copy of the best block's state. pub fn state(&self) -> State { - State::from_existing(self.state_db.lock().unwrap().clone(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce()) + State::from_existing(self.state_db.lock().unwrap().spawn(), HeaderView::new(&self.best_block_header()).state_root(), self.engine.account_start_nonce()) } /// Get info on the cache. @@ -507,7 +507,7 @@ impl Client where V: Verifier { let h = self.chain.read().unwrap().best_block_hash(); let mut b = OpenBlock::new( self.engine.deref().deref(), - self.state_db.lock().unwrap().clone(), + self.state_db.lock().unwrap().spawn(), match self.chain.read().unwrap().block_header(&h) { Some(ref x) => x, None => {return;} }, self.build_last_hashes(h.clone()), self.author(), diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 7c1064abf..9a3d1791d 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -31,7 +31,7 @@ pub type ApplyResult = Result; /// Representation of the entire state of all accounts in the system. pub struct State { - db: JournalDB, + db: Box, root: H256, cache: RefCell>>, snapshots: RefCell>>>>, @@ -41,11 +41,11 @@ pub struct State { impl State { /// Creates new state with empty state root #[cfg(test)] - pub fn new(mut db: JournalDB, account_start_nonce: U256) -> State { + pub fn new(mut db: Box, account_start_nonce: U256) -> State { let mut root = H256::new(); { // init trie and reset root too null - let _ = SecTrieDBMut::new(&mut db, &mut root); + let _ = SecTrieDBMut::new(db.deref_mut(), &mut root); } State { @@ -58,10 +58,10 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: JournalDB, root: H256, account_start_nonce: U256) -> State { + pub fn from_existing(db: Box, root: H256, account_start_nonce: U256) -> State { { // trie should panic! if root does not exist - let _ = SecTrieDB::new(&db, &root); + let _ = SecTrieDB::new(db.as_hashdb(), &root); } State { @@ -126,7 +126,7 @@ impl State { } /// Destroy the current object and return root and database. - pub fn drop(self) -> (H256, JournalDB) { + pub fn drop(self) -> (H256, Box) { (self.root, self.db) } @@ -148,7 +148,7 @@ impl State { /// Determine whether an account exists. pub fn exists(&self, a: &Address) -> bool { - self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(&self.db, &self.root).contains(&a) + self.cache.borrow().get(&a).unwrap_or(&None).is_some() || SecTrieDB::new(self.db.as_hashdb(), &self.root).contains(&a) } /// Get the balance of account `a`. @@ -163,7 +163,7 @@ impl State { /// Mutate storage of account `address` so that it is `value` for `key`. pub fn storage_at(&self, address: &Address, key: &H256) -> H256 { - self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key)) + self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(self.db.as_hashdb(), address), key)) } /// Mutate storage of account `a` so that it is `value` for `key`. @@ -253,7 +253,7 @@ impl State { /// Commits our cached account changes into the trie. pub fn commit(&mut self) { assert!(self.snapshots.borrow().is_empty()); - Self::commit_into(&mut self.db, &mut self.root, self.cache.borrow_mut().deref_mut()); + Self::commit_into(self.db.as_hashdb_mut(), &mut self.root, self.cache.borrow_mut().deref_mut()); } #[cfg(test)] @@ -285,11 +285,11 @@ impl State { fn get<'a>(&'a self, a: &Address, require_code: bool) -> &'a Option { let have_key = self.cache.borrow().contains_key(a); if !have_key { - self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp)) + self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp)) } if require_code { if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() { - account.cache_code(&AccountDB::new(&self.db, a)); + account.cache_code(&AccountDB::new(self.db.as_hashdb(), a)); } } unsafe { ::std::mem::transmute(self.cache.borrow().get(a).unwrap()) } @@ -305,7 +305,7 @@ impl State { fn require_or_from<'a, F: FnOnce() -> Account, G: FnOnce(&mut Account)>(&self, a: &Address, require_code: bool, default: F, not_default: G) -> &'a mut Account { let have_key = self.cache.borrow().contains_key(a); if !have_key { - self.insert_cache(a, SecTrieDB::new(&self.db, &self.root).get(&a).map(Account::from_rlp)) + self.insert_cache(a, SecTrieDB::new(self.db.as_hashdb(), &self.root).get(&a).map(Account::from_rlp)) } else { self.note_cache(a); } @@ -318,7 +318,7 @@ impl State { unsafe { ::std::mem::transmute(self.cache.borrow_mut().get_mut(a).unwrap().as_mut().map(|account| { if require_code { - account.cache_code(&AccountDB::new(&self.db, a)); + account.cache_code(&AccountDB::new(self.db.as_hashdb(), a)); } account }).unwrap()) } diff --git a/ethcore/src/tests/helpers.rs b/ethcore/src/tests/helpers.rs index bb9a44614..7b99f68d5 100644 --- a/ethcore/src/tests/helpers.rs +++ b/ethcore/src/tests/helpers.rs @@ -250,9 +250,9 @@ pub fn generate_dummy_empty_blockchain() -> GuardedTempResult { } } -pub fn get_temp_journal_db() -> GuardedTempResult { +pub fn get_temp_journal_db() -> GuardedTempResult> { let temp = RandomTempPath::new(); - let journal_db = JournalDB::new(temp.as_str()); + let journal_db = Box::new(OptionOneDB::new(temp.as_str())); GuardedTempResult { _temp: temp, result: Some(journal_db) @@ -268,8 +268,8 @@ pub fn get_temp_state() -> GuardedTempResult { } } -pub fn get_temp_journal_db_in(path: &Path) -> JournalDB { - JournalDB::new(path.to_str().unwrap()) +pub fn get_temp_journal_db_in(path: &Path) -> Box { + Box::new(OptionOneDB::new(path.to_str().unwrap())) } pub fn get_temp_state_in(path: &Path) -> State { diff --git a/util/src/hashdb.rs b/util/src/hashdb.rs index 4d8cbaba1..e622c4b99 100644 --- a/util/src/hashdb.rs +++ b/util/src/hashdb.rs @@ -20,7 +20,7 @@ use bytes::*; use std::collections::HashMap; /// Trait modelling datastore keyed by a 32-byte Keccak hash. -pub trait HashDB { +pub trait HashDB : AsHashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; @@ -111,3 +111,16 @@ pub trait HashDB { /// ``` fn remove(&mut self, key: &H256) { self.kill(key) } } + +/// Upcast trait. +pub trait AsHashDB { + /// Perform upcast to HashDB for anything that derives from HashDB. + fn as_hashdb(&self) -> &HashDB; + /// Perform mutable upcast to HashDB for anything that derives from HashDB. + fn as_hashdb_mut(&mut self) -> &mut HashDB; +} + +impl AsHashDB for T { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index 35ad83fa0..23fd08011 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -24,6 +24,22 @@ use kvdb::{Database, DBTransaction, DatabaseConfig}; #[cfg(test)] use std::env; +/// A HashDB which can manage a short-term journal potentially containing many forks of mutually +/// exclusive actions. +pub trait JournalDB : HashDB { + /// Return a copy of ourself, in a box. + fn spawn(&self) -> Box; + + /// Returns heap memory size used + fn mem_used(&self) -> usize; + + /// Check if this database has any commits + fn is_empty(&self) -> bool; + + /// Commit all recent insert operations. + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result; +} + /// Implementation of the HashDB trait for a disk-backed database with a memory overlay /// and latent-removal semantics. /// @@ -31,22 +47,12 @@ use std::env; /// write operations out to disk. Unlike OverlayDB, `remove()` operations do not take effect /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before /// the removals actually take effect. -pub struct JournalDB { +pub struct OptionOneDB { overlay: MemoryDB, backing: Arc, counters: Option>>>, } -impl Clone for JournalDB { - fn clone(&self) -> JournalDB { - JournalDB { - overlay: MemoryDB::new(), - backing: self.backing.clone(), - counters: self.counters.clone(), - } - } -} - // all keys must be at least 12 bytes const LATEST_ERA_KEY : [u8; 12] = [ b'l', b'a', b's', b't', 0, 0, 0, 0, 0, 0, 0, 0 ]; const VERSION_KEY : [u8; 12] = [ b'j', b'v', b'e', b'r', 0, 0, 0, 0, 0, 0, 0, 0 ]; @@ -56,14 +62,14 @@ const DB_VERSION_NO_JOURNAL : u32 = 3 + 256; const PADDING : [u8; 10] = [ 0u8; 10 ]; -impl JournalDB { +impl OptionOneDB { /// Create a new instance from file - pub fn new(path: &str) -> JournalDB { + pub fn new(path: &str) -> OptionOneDB { Self::from_prefs(path, true) } /// Create a new instance from file - pub fn from_prefs(path: &str, prefer_journal: bool) -> JournalDB { + pub fn from_prefs(path: &str, prefer_journal: bool) -> OptionOneDB { let opts = DatabaseConfig { prefix_size: Some(12) //use 12 bytes as prefix, this must match account_db prefix }; @@ -83,11 +89,11 @@ impl JournalDB { } let counters = if with_journal { - Some(Arc::new(RwLock::new(JournalDB::read_counters(&backing)))) + Some(Arc::new(RwLock::new(OptionOneDB::read_counters(&backing)))) } else { None }; - JournalDB { + OptionOneDB { overlay: MemoryDB::new(), backing: Arc::new(backing), counters: counters, @@ -96,27 +102,12 @@ impl JournalDB { /// Create a new instance with an anonymous temporary database. #[cfg(test)] - pub fn new_temp() -> JournalDB { + fn new_temp() -> OptionOneDB { let mut dir = env::temp_dir(); dir.push(H32::random().hex()); Self::new(dir.to_str().unwrap()) } - /// Check if this database has any commits - pub fn is_empty(&self) -> bool { - self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() - } - - /// Commit all recent insert operations. - pub fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { - let have_counters = self.counters.is_some(); - if have_counters { - self.commit_with_counters(now, id, end) - } else { - self.commit_without_counters() - } - } - /// Drain the overlay and place it into a batch for the DB. fn batch_overlay_insertions(overlay: &mut MemoryDB, batch: &DBTransaction) -> usize { let mut inserts = 0usize; @@ -339,11 +330,11 @@ impl JournalDB { try!(batch.delete(&last)); index += 1; } - trace!("JournalDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); + trace!("OptionOneDB: delete journal for time #{}.{}, (canon was {})", end_era, index, canon_id); } try!(self.backing.write(batch)); -// trace!("JournalDB::commit() deleted {} nodes", deletes); +// trace!("OptionOneDB::commit() deleted {} nodes", deletes); Ok(0) } @@ -379,17 +370,9 @@ impl JournalDB { trace!("Recovered {} counters", counters.len()); counters } +} - /// Returns heap memory size used - pub fn mem_used(&self) -> usize { - self.overlay.mem_used() + match self.counters { - Some(ref c) => c.read().unwrap().heap_size_of_children(), - None => 0 - } - } - } - -impl HashDB for JournalDB { +impl HashDB for OptionOneDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); for (key, _) in self.backing.iter() { @@ -434,6 +417,36 @@ impl HashDB for JournalDB { } } +impl JournalDB for OptionOneDB { + fn spawn(&self) -> Box { + Box::new(OptionOneDB { + overlay: MemoryDB::new(), + backing: self.backing.clone(), + counters: self.counters.clone(), + }) + } + + fn mem_used(&self) -> usize { + self.overlay.mem_used() + match self.counters { + Some(ref c) => c.read().unwrap().heap_size_of_children(), + None => 0 + } + } + + fn is_empty(&self) -> bool { + self.backing.get(&LATEST_ERA_KEY).expect("Low level database error").is_none() + } + + fn commit(&mut self, now: u64, id: &H256, end: Option<(u64, H256)>) -> Result { + let have_counters = self.counters.is_some(); + if have_counters { + self.commit_with_counters(now, id, end) + } else { + self.commit_without_counters() + } + } +} + #[cfg(test)] mod tests { use common::*; @@ -443,7 +456,7 @@ mod tests { #[test] fn insert_same_in_fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let x = jdb.insert(b"X"); jdb.commit(1, &b"1".sha3(), None).unwrap(); @@ -465,7 +478,7 @@ mod tests { #[test] fn long_history() { // history is 3 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let h = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); assert!(jdb.exists(&h)); @@ -483,7 +496,7 @@ mod tests { #[test] fn complex() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -521,7 +534,7 @@ mod tests { #[test] fn fork() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -549,7 +562,7 @@ mod tests { #[test] fn overwrite() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -568,7 +581,7 @@ mod tests { #[test] fn fork_same_key() { // history is 1 - let mut jdb = JournalDB::new_temp(); + let mut jdb = OptionOneDB::new_temp(); jdb.commit(0, &b"0".sha3(), None).unwrap(); let foo = jdb.insert(b"foo"); @@ -590,7 +603,7 @@ mod tests { let bar = H256::random(); let foo = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.emplace(bar.clone(), b"bar".to_vec()); @@ -599,13 +612,13 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(1, &b"1".sha3(), Some((0, b"0".sha3()))).unwrap(); } { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); assert!(jdb.exists(&foo)); assert!(jdb.exists(&bar)); jdb.commit(2, &b"2".sha3(), Some((1, b"1".sha3()))).unwrap(); @@ -619,7 +632,7 @@ mod tests { dir.push(H32::random().hex()); let foo = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); jdb.commit(0, &b"0".sha3(), None).unwrap(); @@ -633,7 +646,7 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.remove(&foo); jdb.commit(3, &b"3".sha3(), Some((2, b"2".sha3()))).unwrap(); assert!(jdb.exists(&foo)); @@ -648,7 +661,7 @@ mod tests { let mut dir = ::std::env::temp_dir(); dir.push(H32::random().hex()); let (foo, bar, baz) = { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); // history is 1 let foo = jdb.insert(b"foo"); let bar = jdb.insert(b"bar"); @@ -663,7 +676,7 @@ mod tests { }; { - let mut jdb = JournalDB::new(dir.to_str().unwrap()); + let mut jdb = OptionOneDB::new(dir.to_str().unwrap()); jdb.commit(2, &b"2b".sha3(), Some((1, b"1b".sha3()))).unwrap(); assert!(jdb.exists(&foo)); assert!(!jdb.exists(&baz)); diff --git a/util/src/overlaydb.rs b/util/src/overlaydb.rs index 3c80f4148..f14677d05 100644 --- a/util/src/overlaydb.rs +++ b/util/src/overlaydb.rs @@ -36,6 +36,7 @@ use kvdb::{Database}; /// /// `lookup()` and `contains()` maintain normal behaviour - all `insert()` and `remove()` /// queries have an immediate effect in terms of these functions. +#[derive(Clone)] pub struct OverlayDB { overlay: MemoryDB, backing: Arc,