Refactored JournalDB

ethcore/src/account.rs

@@ -3,6 +3,113 @@
 use util::*;
 use pod_account::*;
 
+
+pub struct AccountDB<'db> {
+	db: &'db HashDB,
+	address: H256,
+}
+
+impl<'db> AccountDB<'db> {
+
+	pub fn new(db: &'db HashDB, address: &Address) -> AccountDB<'db> {
+		AccountDB {
+			db: db,
+			address: x!(address.clone()),
+		}
+	}
+
+	#[inline]
+	fn key(&self, k: &H256) -> H256 {
+		k.clone() ^ self.address.clone()
+	}
+}
+
+impl<'db> HashDB for AccountDB<'db>{
+	fn keys(&self) -> HashMap<H256, i32> {
+		unimplemented!()
+	}
+	fn lookup(&self, key: &H256) -> Option<&[u8]> {
+		if key == &SHA3_NULL_RLP {
+			return self.db.lookup(key);
+		}
+		self.db.lookup(&self.key(key))
+	}
+	fn exists(&self, key: &H256) -> bool {
+		if key == &SHA3_NULL_RLP {
+			return true;
+		}
+		self.db.exists(&self.key(key))
+	}
+	fn insert(&mut self, _value: &[u8]) -> H256 {
+		unimplemented!()
+	}
+
+	fn emplace(&mut self, _key: H256, _value: Bytes) {
+		unimplemented!()
+	}
+	fn kill(&mut self, _key: &H256) {
+		unimplemented!()
+	}
+}
+
+pub struct AccountDBMut<'db> {
+	db: &'db mut HashDB,
+	address: H256,
+}
+
+impl<'db> AccountDBMut<'db> {
+
+	pub fn new(db: &'db mut HashDB, address: &Address) -> AccountDBMut<'db> {
+		AccountDBMut {
+			db: db,
+			address: x!(address.clone()),
+		}
+	}
+
+	#[allow(dead_code)]
+	pub fn immutable(&'db self) -> AccountDB<'db> {
+		AccountDB { db: self.db, address: self.address.clone() }
+	}
+
+	#[inline]
+	fn key(&self, k: &H256) -> H256 {
+		k.clone() ^ self.address.clone()
+	}
+}
+
+impl<'db> HashDB for AccountDBMut<'db>{
+	fn keys(&self) -> HashMap<H256, i32> {
+		unimplemented!()
+	}
+	fn lookup(&self, key: &H256) -> Option<&[u8]> {
+		if key == &SHA3_NULL_RLP {
+			return self.db.lookup(key);
+		}
+		self.db.lookup(&self.key(key))
+	}
+	fn exists(&self, key: &H256) -> bool {
+		if key == &SHA3_NULL_RLP {
+			return true;
+		}
+		self.db.exists(&self.key(key))
+	}
+	fn insert(&mut self, value: &[u8]) -> H256 {
+		let k = value.sha3();
+		let ak = self.key(&k);
+		self.db.emplace(ak, value.to_vec());
+		k
+	}
+
+	fn emplace(&mut self, key: H256, value: Bytes) {
+		let key = self.key(&key);
+		self.db.emplace(key, value.to_vec())
+	}
+	fn kill(&mut self, key: &H256) {
+		let key = self.key(&key);
+		self.db.kill(&key)
+	}
+}
+
 /// Single account in the system.
 #[derive(Clone)]
 pub struct Account {
@@ -99,7 +206,7 @@ impl Account {
 	}
 
 	/// Get (and cache) the contents of the trie's storage at `key`.
-	pub fn storage_at(&self, db: &HashDB, key: &H256) -> H256 {
+	pub fn storage_at(&self, db: &AccountDB, key: &H256) -> H256 {
 		self.storage_overlay.borrow_mut().entry(key.clone()).or_insert_with(||{
 			(Filth::Clean, H256::from(SecTrieDB::new(db, &self.storage_root).get(key.bytes()).map_or(U256::zero(), |v| -> U256 {decode(v)})))
 		}).1.clone()
@@ -147,7 +254,7 @@ impl Account {
 	}
 
 	/// Provide a database to lookup `code_hash`. Should not be called if it is a contract without code.
-	pub fn cache_code(&mut self, db: &HashDB) -> bool {
+	pub fn cache_code(&mut self, db: &AccountDB) -> bool {
 		// TODO: fill out self.code_cache;
 		trace!("Account::cache_code: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty());
 		self.is_cached() ||
@@ -184,7 +291,7 @@ impl Account {
 	pub fn sub_balance(&mut self, x: &U256) { self.balance = self.balance - *x; }
 
 	/// Commit the `storage_overlay` to the backing DB and update `storage_root`.
-	pub fn commit_storage(&mut self, db: &mut HashDB) {
+	pub fn commit_storage(&mut self, db: &mut AccountDBMut) {
 		let mut t = SecTrieDBMut::from_existing(db, &mut self.storage_root);
 		for (k, &mut (ref mut f, ref mut v)) in self.storage_overlay.borrow_mut().iter_mut() {
 			if f == &Filth::Dirty {
@@ -200,7 +307,7 @@ impl Account {
 	}
 
 	/// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this.
-	pub fn commit_code(&mut self, db: &mut HashDB) {
+	pub fn commit_code(&mut self, db: &mut AccountDBMut) {
 		trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_hash.is_none(), self.code_cache.is_empty());
 		match (self.code_hash.is_none(), self.code_cache.is_empty()) {
 			(true, true) => self.code_hash = Some(SHA3_EMPTY),
@@ -237,6 +344,7 @@ mod tests {
 	#[test]
 	fn storage_at() {
 		let mut db = MemoryDB::new();
+		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		let rlp = {
 			let mut a = Account::new_contract(U256::from(69u8));
 			a.set_storage(H256::from(&U256::from(0x00u64)), H256::from(&U256::from(0x1234u64)));
@@ -248,13 +356,14 @@ mod tests {
 
 		let a = Account::from_rlp(&rlp);
 		assert_eq!(a.storage_root().unwrap().hex(), "c57e1afb758b07f8d2c8f13a3b6e44fa5ff94ab266facc5a4fd3f062426e50b2");
-		assert_eq!(a.storage_at(&db, &H256::from(&U256::from(0x00u64))), H256::from(&U256::from(0x1234u64)));
+		assert_eq!(a.storage_at(&db.immutable(), &H256::from(&U256::from(0x00u64))), H256::from(&U256::from(0x1234u64)));
-		assert_eq!(a.storage_at(&db, &H256::from(&U256::from(0x01u64))), H256::new());
+		assert_eq!(a.storage_at(&db.immutable(), &H256::from(&U256::from(0x01u64))), H256::new());
 	}
 
 	#[test]
 	fn note_code() {
 		let mut db = MemoryDB::new();
+		let mut db = AccountDBMut::new(&mut db, &Address::new());
 
 		let rlp = {
 			let mut a = Account::new_contract(U256::from(69u8));
@@ -264,7 +373,7 @@ mod tests {
 		};
 
 		let mut a = Account::from_rlp(&rlp);
-		assert!(a.cache_code(&db));
+		assert!(a.cache_code(&db.immutable()));
 
 		let mut a = Account::from_rlp(&rlp);
 		assert_eq!(a.note_code(vec![0x55, 0x44, 0xffu8]), Ok(()));
@@ -274,6 +383,7 @@ mod tests {
 	fn commit_storage() {
 		let mut a = Account::new_contract(U256::from(69u8));
 		let mut db = MemoryDB::new();
+		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		a.set_storage(x!(0), x!(0x1234));
 		assert_eq!(a.storage_root(), None);
 		a.commit_storage(&mut db);
@@ -284,6 +394,7 @@ mod tests {
 	fn commit_remove_commit_storage() {
 		let mut a = Account::new_contract(U256::from(69u8));
 		let mut db = MemoryDB::new();
+		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		a.set_storage(x!(0), x!(0x1234));
 		a.commit_storage(&mut db);
 		a.set_storage(x!(1), x!(0x1234));
@@ -297,6 +408,7 @@ mod tests {
 	fn commit_code() {
 		let mut a = Account::new_contract(U256::from(69u8));
 		let mut db = MemoryDB::new();
+		let mut db = AccountDBMut::new(&mut db, &Address::new());
 		a.init_code(vec![0x55, 0x44, 0xffu8]);
 		assert_eq!(a.code_hash(), SHA3_EMPTY);
 		a.commit_code(&mut db);

ethcore/src/client.rs

@@ -146,7 +146,7 @@ pub struct Client {
 	import_lock: Mutex<()>
 }
 
-const HISTORY: u64 = 1000;
+const HISTORY: u64 = 1;
 
 impl Client {
 	/// Create a new client with given spec and DB path.
@@ -180,8 +180,7 @@ impl Client {
 		
 		let engine = Arc::new(try!(spec.to_engine()));
 		let mut state_db = JournalDB::new_with_arc(db.clone());
-		if engine.spec().ensure_db_good(&mut state_db) {
+		if engine.spec().ensure_db_good(&mut state_db) { state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB");
-			state_db.commit(0, &engine.spec().genesis_header().hash(), None).expect("Error commiting genesis state to state DB");
 		}
 		Ok(Arc::new(Client {
 			chain: chain,

ethcore/src/pod_account.rs

@@ -44,7 +44,7 @@ impl PodAccount {
 	}
 
 	/// Place additional data into given hash DB.
-	pub fn insert_additional(&self, db: &mut HashDB) {
+	pub fn insert_additional(&self, db: &mut AccountDBMut) {
 		if !self.code.is_empty() {
 			db.insert(&self.code);
 		}

ethcore/src/spec.rs

@@ -262,8 +262,8 @@ impl Spec {
 					t.insert(address.as_slice(), &account.rlp());
 				}
 			}
-			for (_, account) in self.genesis_state.get().iter() {
+			for (address, account) in self.genesis_state.get().iter() {
-				account.insert_additional(db);
+				account.insert_additional(&mut AccountDBMut::new(db, address));
 			}
 			assert!(db.contains(&self.state_root()));
 			true

ethcore/src/state.rs

@@ -144,8 +144,8 @@ impl State {
 	}
 
 	/// Mutate storage of account `a` so that it is `value` for `key`.
-	pub fn storage_at(&self, a: &Address, key: &H256) -> H256 {
+	pub fn storage_at(&self, address: &Address, key: &H256) -> H256 {
-		self.get(a, false).as_ref().map_or(H256::new(), |a|a.storage_at(&self.db, key))	
+		self.get(address, false).as_ref().map_or(H256::new(), |a|a.storage_at(&AccountDB::new(&self.db, address), key))	
 	}
 
 	/// Mutate storage of account `a` so that it is `value` for `key`.
@@ -210,11 +210,12 @@ impl State {
 	pub fn commit_into(db: &mut HashDB, root: &mut H256, accounts: &mut HashMap<Address, Option<Account>>) {
 		// first, commit the sub trees.
 		// TODO: is this necessary or can we dispense with the `ref mut a` for just `a`?
-		for (_, ref mut a) in accounts.iter_mut() {
+		for (address, ref mut a) in accounts.iter_mut() {
 			match a {
 				&mut&mut Some(ref mut account) => {
-					account.commit_storage(db);
+					let mut account_db = AccountDBMut::new(db, address);
-					account.commit_code(db);
+					account.commit_storage(&mut account_db);
+					account.commit_code(&mut account_db);
 				}
 				&mut&mut None => {}
 			}
@@ -270,7 +271,7 @@ impl State {
 		}
 		if require_code {
 			if let Some(ref mut account) = self.cache.borrow_mut().get_mut(a).unwrap().as_mut() {
-				account.cache_code(&self.db);
+				account.cache_code(&AccountDB::new(&self.db, a));
 			}
 		}
 		Ref::map(self.cache.borrow(), |m| m.get(a).unwrap())
@@ -300,7 +301,7 @@ impl State {
 		let b = self.cache.borrow_mut();
 		RefMut::map(b, |m| m.get_mut(a).unwrap().as_mut().map(|account| {
 			if require_code {
-				account.cache_code(&self.db);
+				account.cache_code(&AccountDB::new(&self.db, a));
 			}
 			account
 		}).unwrap())

util/src/journaldb.rs

@@ -3,8 +3,8 @@
 use common::*;
 use rlp::*;
 use hashdb::*;
-use overlaydb::*;
+use memorydb::*;
-use rocksdb::{DB, Writable};
+use rocksdb::{DB, Writable, IteratorMode};
 #[cfg(test)]
 use std::env;
 
@@ -16,10 +16,8 @@ use std::env;
 /// immediately. Rather some age (based on a linear but arbitrary metric) must pass before
 /// the removals actually take effect.
 pub struct JournalDB {
-	forward: OverlayDB,
+	overlay: MemoryDB,
 	backing: Arc<DB>,
-	inserts: Vec<H256>,
-	removes: Vec<H256>,
 }
 
 impl JournalDB {
@@ -27,20 +25,16 @@ impl JournalDB {
 	pub fn new(backing: DB) -> JournalDB {
 		let db = Arc::new(backing);
 		JournalDB {
-			forward: OverlayDB::new_with_arc(db.clone()),
+			overlay: MemoryDB::new(),
 			backing: db,
-			inserts: vec![],
-			removes: vec![],
 		}
 	}
 
 	/// Create a new instance given a shared `backing` database.
 	pub fn new_with_arc(backing: Arc<DB>) -> JournalDB {
 		JournalDB {
-			forward: OverlayDB::new_with_arc(backing.clone()),
+			overlay: MemoryDB::new(),
 			backing: backing,
-			inserts: vec![],
-			removes: vec![],
 		}
 	}
 
@@ -84,12 +78,12 @@ impl JournalDB {
 			}
 
 			let mut r = RlpStream::new_list(3);
+			let inserts: Vec<H256> = self.overlay.keys().iter().filter(|&(_, &c)| c > 0).map(|(key, _)| key.clone()).collect();
+			let removes: Vec<H256> = self.overlay.keys().iter().filter(|&(_, &c)| c < 0).map(|(key, _)| key.clone()).collect();
 			r.append(id);
-			r.append(&self.inserts);
+			r.append(&inserts);
-			r.append(&self.removes);
+			r.append(&removes);
 			try!(self.backing.put(&last, r.as_raw()));
-			self.inserts.clear();
-			self.removes.clear();
 		}
 
 		// apply old commits' details
@@ -106,7 +100,7 @@ impl JournalDB {
 				let rlp = Rlp::new(&rlp_data);
 				let to_remove: Vec<H256> = rlp.val_at(if canon_id == rlp.val_at(0) {2} else {1});
 				for i in &to_remove {
-					self.forward.remove(i);
+					self.backing.delete(&i).expect("Low-level database error. Some issue with your hard disk?");
 				}
 				try!(self.backing.delete(&last));
 				trace!("JournalDB: delete journal for time #{}.{}, (canon was {}): {} entries", end_era, index, canon_id, to_remove.len());
@@ -114,17 +108,88 @@ impl JournalDB {
 			}
 		}
 
-		self.forward.commit()
+		let mut ret = 0u32;
+		let mut deletes = 0usize;
+		for i in self.overlay.drain().into_iter() {
+			let (key, (value, rc)) = i;
+			if rc > 0 {
+				assert!(rc == 1);
+				if !self.backing.get(&key.bytes()).unwrap().is_none() {
+					info!("Exist: {:?}", key);
+					key.clone();
+				}
+				self.backing.put(&key.bytes(), &value).expect("Low-level database error. Some issue with your hard disk?");
+				ret += 1;
+			}
+			if rc < 0 {
+				assert!(rc == -1);
+				ret += 1;
+				deletes += 1;
+			}
+		}
+		trace!("JournalDB::commit() deleted {} nodes", deletes);
+		Ok(ret)
+	}
+
+	fn payload(&self, key: &H256) -> Option<Bytes> {
+		self.backing.get(&key.bytes()).expect("Low-level database error. Some issue with your hard disk?").map(|v| v.to_vec())
 	}
 }
 
 impl HashDB for JournalDB {
-	fn keys(&self) -> HashMap<H256, i32> { self.forward.keys() }
+	fn keys(&self) -> HashMap<H256, i32> { 
-	fn lookup(&self, key: &H256) -> Option<&[u8]> { self.forward.lookup(key) }
+		let mut ret: HashMap<H256, i32> = HashMap::new();
-	fn exists(&self, key: &H256) -> bool { self.forward.exists(key) }
+		for (key, _) in self.backing.iterator(IteratorMode::Start) {
-	fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r }
+			let h = H256::from_slice(key.deref());
-	fn emplace(&mut self, key: H256, value: Bytes) { self.inserts.push(key.clone()); self.forward.emplace(key, value); }
+			ret.insert(h, 1);
-	fn kill(&mut self, key: &H256) { self.removes.push(key.clone()); }
+		}
+
+		for (key, refs) in self.overlay.keys().into_iter() {
+			let refs = *ret.get(&key).unwrap_or(&0) + refs;
+			ret.insert(key, refs);
+		}
+		ret
+	}
+
+	fn lookup(&self, key: &H256) -> Option<&[u8]> { 
+		let k = self.overlay.raw(key);
+		match k {
+			Some(&(ref d, rc)) if rc > 0 => Some(d),
+			_ => {
+				if let Some(x) = self.payload(key) {
+					Some(&self.overlay.denote(key, x).0)
+				}
+				else {
+					None
+				}
+			}
+		}
+	}
+
+	fn exists(&self, key: &H256) -> bool { 
+		self.lookup(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H256 { 
+		if value.sha3() == h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") {
+			info!("GOTCHA");
+			value.to_vec();
+		}
+		self.overlay.insert(value) 
+	}
+	fn emplace(&mut self, key: H256, value: Bytes) {
+		if key == h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") {
+			info!("GOTCHA");
+			value.to_vec();
+		}
+		self.overlay.emplace(key, value); 
+	}
+	fn kill(&mut self, key: &H256) { 
+			if key == &h256_from_hex("3567da57862169b0dc409933ec10da8113ef3810fd225ad81d4fc23c36ffa5d4") {
+			info!("DELETING");
+			key.clone();
+		}
+		self.overlay.kill(key); }
 }
 
 #[cfg(test)]

util/src/trie/triedb.rs

@@ -196,7 +196,7 @@ impl<'db> TrieDB<'db> {
 		// check if its sha3 + len
 		let r = Rlp::new(node);
 		match r.is_data() && r.size() == 32 {
-			true => self.db.lookup(&r.as_val::<H256>()).expect("Not found!"),
+			true => self.db.lookup(&r.as_val::<H256>()).unwrap_or_else(|| panic!("Not found! {:?}", r.as_val::<H256>())),
 			false => node
 		}
 	}