From 6b3390f03473e6ad21f9c14a4822f02d3c58519c Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 4 Feb 2016 02:40:35 +0100 Subject: [PATCH] Refactored JournalDB --- ethcore/src/account.rs | 126 ++++++++++++++++++++++++++++++++++--- ethcore/src/client.rs | 5 +- ethcore/src/pod_account.rs | 2 +- ethcore/src/spec.rs | 4 +- ethcore/src/state.rs | 15 ++--- util/src/journaldb.rs | 111 +++++++++++++++++++++++++------- util/src/trie/triedb.rs | 2 +- 7 files changed, 221 insertions(+), 44 deletions(-) diff --git a/ethcore/src/account.rs b/ethcore/src/account.rs index 63f86b171..bde8eb8b7 100644 --- a/ethcore/src/account.rs +++ b/ethcore/src/account.rs @@ -3,6 +3,113 @@ use util::*; use pod_account::*; + +pub struct AccountDB<'db> { + db: &'db HashDB, + address: H256, +} + +impl<'db> AccountDB<'db> { + + pub fn new(db: &'db HashDB, address: &Address) -> AccountDB<'db> { + AccountDB { + db: db, + address: x!(address.clone()), + } + } + + #[inline] + fn key(&self, k: &H256) -> H256 { + k.clone() ^ self.address.clone() + } +} + +impl<'db> HashDB for AccountDB<'db>{ + fn keys(&self) -> HashMap { + unimplemented!() + } + fn lookup(&self, key: &H256) -> Option<&[u8]> { + if key == &SHA3_NULL_RLP { + return self.db.lookup(key); + } + self.db.lookup(&self.key(key)) + } + fn exists(&self, key: &H256) -> bool { + if key == &SHA3_NULL_RLP { + return true; + } + self.db.exists(&self.key(key)) + } + fn insert(&mut self, _value: &[u8]) -> H256 { + unimplemented!() + } + + fn emplace(&mut self, _key: H256, _value: Bytes) { + unimplemented!() + } + fn kill(&mut self, _key: &H256) { + unimplemented!() + } +} + +pub struct AccountDBMut<'db> { + db: &'db mut HashDB, + address: H256, +} + +impl<'db> AccountDBMut<'db> { + + pub fn new(db: &'db mut HashDB, address: &Address) -> AccountDBMut<'db> { + AccountDBMut { + db: db, + address: x!(address.clone()), + } + } + + #[allow(dead_code)] + pub fn immutable(&'db self) -> AccountDB<'db> { + AccountDB { db: self.db, address: self.address.clone() } + } + + #[inline] + fn key(&self, k: &H256) -> H256 { + k.clone() ^ self.address.clone() + } +} + +impl<'db> HashDB for AccountDBMut<'db>{ + fn keys(&self) -> HashMap { + unimplemented!() + } + fn lookup(&self, key: &H256) -> Option<&[u8]> { + if key == &SHA3_NULL_RLP { + return self.db.lookup(key); + } + self.db.lookup(&self.key(key)) + } + fn exists(&self, key: &H256) -> bool { + if key == &SHA3_NULL_RLP { + return true; + } + self.db.exists(&self.key(key)) + } + fn insert(&mut self, value: &[u8]) -> H256 { + let k = value.sha3(); + let ak = self.key(&k); + self.db.emplace(ak, value.to_vec()); + k + } + + fn emplace(&mut self, key: H256, value: Bytes) { + let key = self.key(&key); + self.db.emplace(key, value.to_vec()) + } + fn kill(&mut self, key: &H256) { + let key = self.key(&key); + self.db.kill(&key) + } +} + /// Single account in the system. #[derive(Clone)] pub struct Account { @@ -99,7 +206,7 @@ impl Account { } /// Get (and cache) the contents of the trie's storage at `key`. - pub fn storage_at(&self, db: &HashDB, key: &H256) -> H256 { + pub fn storage_at(&self, db: &AccountDB, key: &H256) -> H256 { self.storage_overlay.borrow_mut().entry(key.clone()).or_insert_with(||{ (Filth::Clean, H256::from(SecTrieDB::new(db, &self.storage_root).get(key.bytes()).map_or(U256::zero(), |v| -> U256 {decode(v)}))) }).1.clone() @@ -147,7 +254,7 @@ impl Account { } /// Provide a database to lookup `code_hash`. Should not be called if it is a contract without code. - pub fn cache_code(&mut self, db: &HashDB) -> bool { + pub fn cache_code(&mut self, db: &AccountDB) -> bool { // TODO: fill out self.code_cache; trace!("Account::cache_code: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); self.is_cached() || @@ -184,7 +291,7 @@ impl Account { pub fn sub_balance(&mut self, x: &U256) { self.balance = self.balance - *x; } /// Commit the `storage_overlay` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, db: &mut HashDB) { + pub fn commit_storage(&mut self, db: &mut AccountDBMut) { let mut t = SecTrieDBMut::from_existing(db, &mut self.storage_root); for (k, &mut (ref mut f, ref mut v)) in self.storage_overlay.borrow_mut().iter_mut() { if f == &Filth::Dirty { @@ -200,7 +307,7 @@ impl Account { } /// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this. - pub fn commit_code(&mut self, db: &mut HashDB) { + pub fn commit_code(&mut self, db: &mut AccountDBMut) { trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty()); match (self.code_hash.is_none(), self.code_cache.is_empty()) { (true, true) => self.code_hash = Some(SHA3_EMPTY), @@ -237,6 +344,7 @@ mod tests { #[test] fn storage_at() { let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); let rlp = { let mut a = Account::new_contract(U256::from(69u8)); a.set_storage(H256::from(&U256::from(0x00u64)), H256::from(&U256::from(0x1234u64))); @@ -248,13 +356,14 @@ mod tests { let a = Account::from_rlp(&rlp); assert_eq!(a.storage_root().unwrap().hex(), "c57e1afb758b07f8d2c8f13a3b6e44fa5ff94ab266facc5a4fd3f062426e50b2"); - assert_eq!(a.storage_at(&db, &H256::from(&U256::from(0x00u64))), H256::from(&U256::from(0x1234u64))); - assert_eq!(a.storage_at(&db, &H256::from(&U256::from(0x01u64))), H256::new()); + assert_eq!(a.storage_at(&db.immutable(), &H256::from(&U256::from(0x00u64))), H256::from(&U256::from(0x1234u64))); + assert_eq!(a.storage_at(&db.immutable(), &H256::from(&U256::from(0x01u64))), H256::new()); } #[test] fn note_code() { let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); let rlp = { let mut a = Account::new_contract(U256::from(69u8)); @@ -264,7 +373,7 @@ mod tests { }; let mut a = Account::from_rlp(&rlp); - assert!(a.cache_code(&db)); + assert!(a.cache_code(&db.immutable())); let mut a = Account::from_rlp(&rlp); assert_eq!(a.note_code(vec![0x55, 0x44, 0xffu8]), Ok(())); @@ -274,6 +383,7 @@ mod tests { fn commit_storage() { let mut a = Account::new_contract(U256::from(69u8)); let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); a.set_storage(x!(0), x!(0x1234)); assert_eq!(a.storage_root(), None); a.commit_storage(&mut db); @@ -284,6 +394,7 @@ mod tests { fn commit_remove_commit_storage() { let mut a = Account::new_contract(U256::from(69u8)); let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); a.set_storage(x!(0), x!(0x1234)); a.commit_storage(&mut db); a.set_storage(x!(1), x!(0x1234)); @@ -297,6 +408,7 @@ mod tests { fn commit_code() { let mut a = Account::new_contract(U256::from(69u8)); let mut db = MemoryDB::new(); + let mut db = AccountDBMut::new(&mut db, &Address::new()); a.init_code(vec![0x55, 0x44, 0xffu8]); assert_eq!(a.code_hash(), SHA3_EMPTY); a.commit_code(&mut db); diff --git a/ethcore/src/client.rs b/ethcore/src/client.rs index f5fd0f8f5..71c27502b 100644 --- a/ethcore/src/client.rs +++ b/ethcore/src/client.rs @@ -146,7 +146,7 @@ pub struct Client { import_lock: Mutex<()> } -const HISTORY: u64 = 1000; +const HISTORY: u64 = 1; impl Client { /// Create a new client with given spec and DB path. @@ -180,8 +180,7 @@ impl Client { let engine = Arc::new(try!(spec.to_engine())); let mut state_db = JournalDB::new_with_arc(db.clone()); - if engine.spec().ensure_db_good(&mut state_db) { - state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); + if engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB"); } Ok(Arc::new(Client { chain: chain, diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 7bc886617..1c6142610 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -44,7 +44,7 @@ impl PodAccount { } /// Place additional data into given hash DB. - pub fn insert_additional(&self, db: &mut HashDB) { + pub fn insert_additional(&self, db: &mut AccountDBMut) { if !self.code.is_empty() { db.insert(&self.code); } diff --git a/ethcore/src/spec.rs b/ethcore/src/spec.rs index 50bbbb633..2b291249e 100644 --- a/ethcore/src/spec.rs +++ b/ethcore/src/spec.rs @@ -262,8 +262,8 @@ impl Spec { t.insert(address.as_slice(), &account.rlp()); } } - for (_, account) in self.genesis_state.get().iter() { - account.insert_additional(db); + for (address, account) in self.genesis_state.get().iter() { + account.insert_additional(&mut AccountDBMut::new(db, address)); } assert!(db.contains(&self.state_root())); true diff --git a/ethcore/src/state.rs b/ethcore/src/state.rs index 3573897ca..d80eed1f3 100644 --- a/ethcore/src/state.rs +++ b/ethcore/src/state.rs @@ -144,8 +144,8 @@ impl State { } /// Mutate storage of account `a` so that it is `value` for `key`. - pub fn storage_at(&self, a: &Address, key: &H256) -> H256 { - self.get(a, false).as_ref().map_or(H256::new(), |a|a.storage_at(&self.db, key)) + pub fn storage_at(&self, address: &Address, key: &H256) -> H256 { + self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key)) } /// Mutate storage of account `a` so that it is `value` for `key`. @@ -210,11 +210,12 @@ impl State { pub fn commit_into(db: &mut HashDB, root: &mut H256, accounts: &mut HashMap>) { // first, commit the sub trees. // TODO: is this necessary or can we dispense with the `ref mut a` for just `a`? - for (_, ref mut a) in accounts.iter_mut() { + for (address, ref mut a) in accounts.iter_mut() { match a { &mut&mut Some(ref mut account) => { - account.commit_storage(db); - account.commit_code(db); + let mut account_db = AccountDBMut::new(db, address); + account.commit_storage(&mut account_db); + account.commit_code(&mut account_db); } &mut&mut None => {} } @@ -270,7 +271,7 @@ impl State { } if require_code { if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() { - account.cache_code(&self.db); + account.cache_code(&AccountDB::new(&self.db, a)); } } Ref::map(self.cache.borrow(), |m| m.get(a).unwrap()) @@ -300,7 +301,7 @@ impl State { let b = self.cache.borrow_mut(); RefMut::map(b, |m| m.get_mut(a).unwrap().as_mut().map(|account| { if require_code { - account.cache_code(&self.db); + account.cache_code(&AccountDB::new(&self.db, a)); } account }).unwrap()) diff --git a/util/src/journaldb.rs b/util/src/journaldb.rs index a8de1f71f..a07a9537d 100644 --- a/util/src/journaldb.rs +++ b/util/src/journaldb.rs @@ -3,8 +3,8 @@ use common::*; use rlp::*; use hashdb::*; -use overlaydb::*; -use rocksdb::{DB, Writable}; +use memorydb::*; +use rocksdb::{DB, Writable, IteratorMode}; #[cfg(test)] use std::env; @@ -16,10 +16,8 @@ use std::env; /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before /// the removals actually take effect. pub struct JournalDB { - forward: OverlayDB, + overlay: MemoryDB, backing: Arc, - inserts: Vec, - removes: Vec, } impl JournalDB { @@ -27,20 +25,16 @@ impl JournalDB { pub fn new(backing: DB) -> JournalDB { let db = Arc::new(backing); JournalDB { - forward: OverlayDB::new_with_arc(db.clone()), + overlay: MemoryDB::new(), backing: db, - inserts: vec![], - removes: vec![], } } /// Create a new instance given a shared `backing` database. pub fn new_with_arc(backing: Arc) -> JournalDB { JournalDB { - forward: OverlayDB::new_with_arc(backing.clone()), + overlay: MemoryDB::new(), backing: backing, - inserts: vec![], - removes: vec![], } } @@ -84,12 +78,12 @@ impl JournalDB { } let mut r = RlpStream::new_list(3); + let inserts: Vec = self.overlay.keys().iter().filter(|&(_, &c)| c > 0).map(|(key, _)| key.clone()).collect(); + let removes: Vec = self.overlay.keys().iter().filter(|&(_, &c)| c < 0).map(|(key, _)| key.clone()).collect(); r.append(id); - r.append(&self.inserts); - r.append(&self.removes); + r.append(&inserts); + r.append(&removes); try!(self.backing.put(&last, r.as_raw())); - self.inserts.clear(); - self.removes.clear(); } // apply old commits' details @@ -106,7 +100,7 @@ impl JournalDB { let rlp = Rlp::new(&rlp_data); let to_remove: Vec = rlp.val_at(if canon_id == rlp.val_at(0) {2} else {1}); for i in &to_remove { - self.forward.remove(i); + self.backing.delete(&i).expect("Low-level database error. Some issue with your hard disk?"); } try!(self.backing.delete(&last)); trace!("JournalDB: delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, to_remove.len()); @@ -114,17 +108,88 @@ impl JournalDB { } } - self.forward.commit() + let mut ret = 0u32; + let mut deletes = 0usize; + for i in self.overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc > 0 { + assert!(rc == 1); + if !self.backing.get(&key.bytes()).unwrap().is_none() { + info!("Exist: {:?}", key); + key.clone(); + } + self.backing.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?"); + ret += 1; + } + if rc < 0 { + assert!(rc == -1); + ret += 1; + deletes += 1; + } + } + trace!("JournalDB::commit() deleted {} nodes", deletes); + Ok(ret) + } + + fn payload(&self, key: &H256) -> Option { + self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec()) } } impl HashDB for JournalDB { - fn keys(&self) -> HashMap { self.forward.keys() } - fn lookup(&self, key: &H256) -> Option<&[u8]> { self.forward.lookup(key) } - fn exists(&self, key: &H256) -> bool { self.forward.exists(key) } - fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r } - fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); } - fn kill(&mut self, key: &H256) { self.removes.push(key.clone()); } + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); + for (key, _) in self.backing.iterator(IteratorMode::Start) { + let h = H256::from_slice(key.deref()); + ret.insert(h, 1); + } + + for (key, refs) in self.overlay.keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0) + refs; + ret.insert(key, refs); + } + ret + } + + fn lookup(&self, key: &H256) -> Option<&[u8]> { + let k = self.overlay.raw(key); + match k { + Some(&(ref d, rc)) if rc > 0 => Some(d), + _ => { + if let Some(x) = self.payload(key) { + Some(&self.overlay.denote(key, x).0) + } + else { + None + } + } + } + } + + fn exists(&self, key: &H256) -> bool { + self.lookup(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> H256 { + if value.sha3() == h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") { + info!("GOTCHA"); + value.to_vec(); + } + self.overlay.insert(value) + } + fn emplace(&mut self, key: H256, value: Bytes) { + if key == h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") { + info!("GOTCHA"); + value.to_vec(); + } + self.overlay.emplace(key, value); + } + fn kill(&mut self, key: &H256) { + if key == &h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") { + info!("DELETING"); + key.clone(); + } + self.overlay.kill(key); } } #[cfg(test)] diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 9909c05a4..14b927149 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -196,7 +196,7 @@ impl<'db> TrieDB<'db> { // check if its sha3 + len let r = Rlp::new(node); match r.is_data() && r.size() == 32 { - true => self.db.lookup(&r.as_val::()).expect("Not found!"), + true => self.db.lookup(&r.as_val::()).unwrap_or_else(|| panic!("Not found! {:?}", r.as_val::())), false => node } }