More performance optimizations (#1814)

* Buffered DB

* Use identity hash for MemoryDB

* Various tweaks

* Delayed DB compression

* Reduce last_hashes cloning

* Keep state cache

* Updating tests

* Optimized to_big_int

* Fixing build with stable

* Safer code

ethcore/src/account.rs

@@ -191,6 +191,12 @@ impl Account {
 	pub fn is_dirty(&self) -> bool {
 		self.filth == Filth::Dirty
 	}
+
+	/// Mark account as clean.
+	pub fn set_clean(&mut self) {
+		self.filth = Filth::Clean
+	}
+
 	/// Provide a database to get `code_hash`. Should not be called if it is a contract without code.
 	pub fn cache_code(&mut self, db: &AccountDB) -> bool {
 		// TODO: fill out self.code_cache;

ethcore/src/block.rs

@@ -193,7 +193,7 @@ pub struct OpenBlock<'x> {
 	block: ExecutedBlock,
 	engine: &'x Engine,
 	vm_factory: &'x EvmFactory,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 }
 
 /// Just like `OpenBlock`, except that we've applied `Engine::on_close_block`, finished up the non-seal header fields,
@@ -204,7 +204,7 @@ pub struct OpenBlock<'x> {
 pub struct ClosedBlock {
 	block: ExecutedBlock,
 	uncle_bytes: Bytes,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 	unclosed_state: State,
 }
 
@@ -235,7 +235,7 @@ impl<'x> OpenBlock<'x> {
 		tracing: bool,
 		db: Box<JournalDB>,
 		parent: &Header,
-		last_hashes: LastHashes,
+		last_hashes: Arc<LastHashes>,
 		author: Address,
 		gas_range_target: (U256, U256),
 		extra_data: Bytes,
@@ -316,7 +316,7 @@ impl<'x> OpenBlock<'x> {
 			author: self.block.base.header.author.clone(),
 			timestamp: self.block.base.header.timestamp,
 			difficulty: self.block.base.header.difficulty.clone(),
-			last_hashes: self.last_hashes.clone(),		// TODO: should be a reference.
+			last_hashes: self.last_hashes.clone(),
 			gas_used: self.block.receipts.last().map_or(U256::zero(), |r| r.gas_used),
 			gas_limit: self.block.base.header.gas_limit.clone(),
 		}
@@ -498,7 +498,7 @@ pub fn enact(
 	tracing: bool,
 	db: Box<JournalDB>,
 	parent: &Header,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 	vm_factory: &EvmFactory,
 	trie_factory: TrieFactory,
 ) -> Result<LockedBlock, Error> {
@@ -531,7 +531,7 @@ pub fn enact_bytes(
 	tracing: bool,
 	db: Box<JournalDB>,
 	parent: &Header,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 	vm_factory: &EvmFactory,
 	trie_factory: TrieFactory,
 ) -> Result<LockedBlock, Error> {
@@ -548,7 +548,7 @@ pub fn enact_verified(
 	tracing: bool,
 	db: Box<JournalDB>,
 	parent: &Header,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 	vm_factory: &EvmFactory,
 	trie_factory: TrieFactory,
 ) -> Result<LockedBlock, Error> {
@@ -564,7 +564,7 @@ pub fn enact_and_seal(
 	tracing: bool,
 	db: Box<JournalDB>,
 	parent: &Header,
-	last_hashes: LastHashes,
+	last_hashes: Arc<LastHashes>,
 	vm_factory: &EvmFactory,
 	trie_factory: TrieFactory,
 ) -> Result<SealedBlock, Error> {
@@ -587,7 +587,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let last_hashes = vec![genesis_header.hash()];
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let vm_factory = Default::default();
 		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();
@@ -605,7 +605,8 @@ mod tests {
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
 		let vm_factory = Default::default();
-		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, vec![genesis_header.hash()], Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap()
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
+		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap()
 			.close_and_lock().seal(engine.deref(), vec![]).unwrap();
 		let orig_bytes = b.rlp_bytes();
 		let orig_db = b.drain();
@@ -613,7 +614,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let e = enact_and_seal(&orig_bytes, engine.deref(), false, db, &genesis_header, vec![genesis_header.hash()], &Default::default(), Default::default()).unwrap();
+		let e = enact_and_seal(&orig_bytes, engine.deref(), false, db, &genesis_header, last_hashes, &Default::default(), Default::default()).unwrap();
 
 		assert_eq!(e.rlp_bytes(), orig_bytes);
 
@@ -633,7 +634,8 @@ mod tests {
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
 		let vm_factory = Default::default();
-		let mut open_block = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, vec![genesis_header.hash()], Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
+		let mut open_block = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes.clone(), Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle1_header = Header::new();
 		uncle1_header.extra_data = b"uncle1".to_vec();
 		let mut uncle2_header = Header::new();
@@ -648,7 +650,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let e = enact_and_seal(&orig_bytes, engine.deref(), false, db, &genesis_header, vec![genesis_header.hash()], &Default::default(), Default::default()).unwrap();
+		let e = enact_and_seal(&orig_bytes, engine.deref(), false, db, &genesis_header, last_hashes, &Default::default(), Default::default()).unwrap();
 
 		let bytes = e.rlp_bytes();
 		assert_eq!(bytes, orig_bytes);

ethcore/src/blockchain/blockchain.rs

@@ -549,15 +549,11 @@ impl BlockChain {
 
 		assert!(self.pending_best_block.read().is_none());
 
-		let block_rlp = UntrustedRlp::new(bytes);
-		let compressed_header = block_rlp.at(0).unwrap().compress(RlpType::Blocks);
-		let compressed_body = UntrustedRlp::new(&Self::block_to_body(bytes)).compress(RlpType::Blocks);
-
 		// store block in db
-		batch.put(DB_COL_HEADERS, &hash, &compressed_header).unwrap();
-		batch.put(DB_COL_BODIES, &hash, &compressed_body).unwrap();
+		batch.put_compressed(DB_COL_HEADERS, &hash, block.header_rlp().as_raw().to_vec()).unwrap();
+		batch.put_compressed(DB_COL_BODIES, &hash, Self::block_to_body(bytes)).unwrap();
 
-		let info = self.block_info(bytes);
+		let info = self.block_info(&header);
 
 		if let BlockLocation::BranchBecomingCanonChain(ref d) = info.location {
 			info!(target: "reorg", "Reorg to {} ({} {} {})",
@@ -582,10 +578,8 @@ impl BlockChain {
 	}
 
 	/// Get inserted block info which is critical to prepare extras updates.
-	fn block_info(&self, block_bytes: &[u8]) -> BlockInfo {
-		let block = BlockView::new(block_bytes);
-		let header = block.header_view();
-		let hash = block.sha3();
+	fn block_info(&self, header: &HeaderView) -> BlockInfo {
+		let hash = header.sha3();
 		let number = header.number();
 		let parent_hash = header.parent_hash();
 		let parent_details = self.block_details(&parent_hash).unwrap_or_else(|| panic!("Invalid parent hash: {:?}", parent_hash));

ethcore/src/client/client.rs

@@ -246,13 +246,13 @@ impl Client {
 		}
 	}
 
-	fn build_last_hashes(&self, parent_hash: H256) -> LastHashes {
+	fn build_last_hashes(&self, parent_hash: H256) -> Arc<LastHashes> {
 		{
 			let hashes = self.last_hashes.read();
 			if hashes.front().map_or(false, |h| h == &parent_hash) {
 				let mut res = Vec::from(hashes.clone());
 				res.resize(256, H256::default());
-				return res;
+				return Arc::new(res);
 			}
 		}
 		let mut last_hashes = LastHashes::new();
@@ -268,7 +268,7 @@ impl Client {
 		}
 		let mut cached_hashes = self.last_hashes.write();
 		*cached_hashes = VecDeque::from(last_hashes.clone());
-		last_hashes
+		Arc::new(last_hashes)
 	}
 
 	fn check_and_close_block(&self, block: &PreverifiedBlock) -> Result<LockedBlock, ()> {
@@ -413,6 +413,7 @@ impl Client {
 			}
 		}
 
+		self.db.flush().expect("DB flush failed.");
 		imported
 	}
 
@@ -440,7 +441,7 @@ impl Client {
 		// CHECK! I *think* this is fine, even if the state_root is equal to another
 		// already-imported block of the same number.
 		// TODO: Prove it with a test.
-		block.drain().commit(&batch, number, hash, ancient).expect("State DB commit failed.");
+		block.drain().commit(&batch, number, hash, ancient).expect("DB commit failed.");
 
 		let route = self.chain.insert_block(&batch, block_data, receipts);
 		self.tracedb.import(&batch, TraceImportRequest {
@@ -451,7 +452,7 @@ impl Client {
 			retracted: route.retracted.len()
 		});
 		// Final commit to the DB
-		self.db.write(batch).expect("State DB write failed.");
+		self.db.write_buffered(batch).expect("DB write failed.");
 		self.chain.commit();
 
 		self.update_last_hashes(&parent, hash);
@@ -975,7 +976,7 @@ impl BlockChainClient for Client {
 	}
 
 	fn last_hashes(&self) -> LastHashes {
-		self.build_last_hashes(self.chain.best_block_hash())
+		(*self.build_last_hashes(self.chain.best_block_hash())).clone()
 	}
 
 	fn queue_transactions(&self, transactions: Vec<Bytes>) {
@@ -1059,6 +1060,7 @@ impl MiningBlockChainClient for Client {
 				precise_time_ns() - start,
 			);
 		});
+		self.db.flush().expect("DB flush failed.");
 		Ok(h)
 	}
 }

ethcore/src/client/test_client.rs

@@ -272,7 +272,7 @@ impl MiningBlockChainClient for TestBlockChainClient {
 			false,
 			db,
 			&genesis_header,
-			last_hashes,
+			Arc::new(last_hashes),
 			author,
 			gas_range_target,
 			extra_data

ethcore/src/engines/basic_authority.rs

@@ -202,7 +202,7 @@ mod tests {
 			author: 0.into(),
 			timestamp: 0,
 			difficulty: 0.into(),
-			last_hashes: vec![],
+			last_hashes: Arc::new(vec![]),
 			gas_used: 0.into(),
 			gas_limit: 0.into(),
 		});
@@ -251,7 +251,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let last_hashes = vec![genesis_header.hash()];
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let vm_factory = Default::default();
 		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/engines/instant_seal.rs

@@ -85,7 +85,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let last_hashes = vec![genesis_header.hash()];
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let vm_factory = Default::default();
 		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes, addr, (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close_and_lock();

ethcore/src/env_info.rs

@@ -36,7 +36,7 @@ pub struct EnvInfo {
 	/// The block gas limit.
 	pub gas_limit: U256,
 	/// The last 256 block hashes.
-	pub last_hashes: LastHashes,
+	pub last_hashes: Arc<LastHashes>,
 	/// The gas used.
 	pub gas_used: U256,
 }
@@ -49,7 +49,7 @@ impl Default for EnvInfo {
 			timestamp: 0,
 			difficulty: 0.into(),
 			gas_limit: 0.into(),
-			last_hashes: vec![],
+			last_hashes: Arc::new(vec![]),
 			gas_used: 0.into(),
 		}
 	}
@@ -64,7 +64,7 @@ impl From<ethjson::vm::Env> for EnvInfo {
 			difficulty: e.difficulty.into(),
 			gas_limit: e.gas_limit.into(),
 			timestamp: e.timestamp.into(),
-			last_hashes: (1..cmp::min(number + 1, 257)).map(|i| format!("{}", number - i).as_bytes().sha3()).collect(),
+			last_hashes: Arc::new((1..cmp::min(number + 1, 257)).map(|i| format!("{}", number - i).as_bytes().sha3()).collect()),
 			gas_used: U256::zero(),
 		}
 	}

ethcore/src/ethereum/ethash.rs

@@ -355,7 +355,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let last_hashes = vec![genesis_header.hash()];
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let vm_factory = Default::default();
 		let b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let b = b.close();
@@ -370,7 +370,7 @@ mod tests {
 		let mut db_result = get_temp_journal_db();
 		let mut db = db_result.take();
 		spec.ensure_db_good(db.as_hashdb_mut()).unwrap();
-		let last_hashes = vec![genesis_header.hash()];
+		let last_hashes = Arc::new(vec![genesis_header.hash()]);
 		let vm_factory = Default::default();
 		let mut b = OpenBlock::new(engine.deref(), &vm_factory, Default::default(), false, db, &genesis_header, last_hashes, Address::zero(), (3141562.into(), 31415620.into()), vec![]).unwrap();
 		let mut uncle = Header::new();
@@ -398,7 +398,7 @@ mod tests {
 			author: 0.into(),
 			timestamp: 0,
 			difficulty: 0.into(),
-			last_hashes: vec![],
+			last_hashes: Arc::new(vec![]),
 			gas_used: 0.into(),
 			gas_limit: 0.into(),
 		});
@@ -410,7 +410,7 @@ mod tests {
 			author: 0.into(),
 			timestamp: 0,
 			difficulty: 0.into(),
-			last_hashes: vec![],
+			last_hashes: Arc::new(vec![]),
 			gas_used: 0.into(),
 			gas_limit: 0.into(),
 		});

ethcore/src/externalities.rs

@@ -324,7 +324,7 @@ mod tests {
 			author: 0.into(),
 			timestamp: 0,
 			difficulty: 0.into(),
-			last_hashes: vec![],
+			last_hashes: Arc::new(vec![]),
 			gas_used: 0.into(),
 			gas_limit: 0.into(),
 		}
@@ -391,7 +391,9 @@ mod tests {
 		{
 			let env_info = &mut setup.env_info;
 			env_info.number = test_env_number;
-			env_info.last_hashes.push(test_hash.clone());
+			let mut last_hashes = (*env_info.last_hashes).clone();
+			last_hashes.push(test_hash.clone());
+			env_info.last_hashes = Arc::new(last_hashes);
 		}
 		let state = setup.state.reference_mut();
 		let mut tracer = NoopTracer;

ethcore/src/miner/miner.rs

@@ -472,7 +472,7 @@ impl MinerService for Miner {
 
 				// TODO: merge this code with client.rs's fn call somwhow.
 				let header = block.header();
-				let last_hashes = chain.last_hashes();
+				let last_hashes = Arc::new(chain.last_hashes());
 				let env_info = EnvInfo {
 					number: header.number(),
 					author: *header.author(),

ethcore/src/state.rs

@@ -163,9 +163,7 @@ impl State {
 
 	/// Determine whether an account exists.
 	pub fn exists(&self, a: &Address) -> bool {
-		let db = self.trie_factory.readonly(self.db.as_hashdb(), &self.root).expect(SEC_TRIE_DB_UNWRAP_STR);
-		self.cache.borrow().get(&a).unwrap_or(&None).is_some() ||
-			db.contains(a).unwrap_or_else(|e| { warn!("Potential DB corruption encountered: {}", e); false })
+		self.ensure_cached(a, false, |a| a.is_some())
 	}
 
 	/// Get the balance of account `a`.
@@ -242,7 +240,6 @@ impl State {
 		// TODO uncomment once to_pod() works correctly.
 //		trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod()));
 		try!(self.commit());
-		self.clear();
 		let receipt = Receipt::new(self.root().clone(), e.cumulative_gas_used, e.logs);
 //		trace!("Transaction receipt: {:?}", receipt);
 		Ok(ApplyOutcome{receipt: receipt, trace: e.trace})
@@ -273,9 +270,12 @@ impl State {
 
 		{
 			let mut trie = trie_factory.from_existing(db, root).unwrap();
-			for (address, ref a) in accounts.iter() {
+			for (address, ref mut a) in accounts.iter_mut() {
 				match **a {
-					Some(ref account) if account.is_dirty() => try!(trie.insert(address, &account.rlp())),
+					Some(ref mut account) if account.is_dirty() => {
+						account.set_clean();
+						try!(trie.insert(address, &account.rlp()))
+					},
 					None => try!(trie.remove(address)),
 					_ => (),
 				}

ethcore/src/tests/helpers.rs

@@ -160,7 +160,7 @@ pub fn generate_dummy_client_with_spec_and_data<F>(get_test_spec: F, block_numbe
 			false,
 			db,
 			&last_header,
-			last_hashes.clone(),
+			Arc::new(last_hashes.clone()),
 			author.clone(),
 			(3141562.into(), 31415620.into()),
 			vec![]