Configurable history size in beta (#2587)

* Reduce DB history

* Configurable history size

* Set min history size

* Test

* Fixed a test and reduced the limit

ethcore/src/client/client.rs

@@ -17,6 +17,7 @@ use std::collections::{HashSet, HashMap, BTreeMap, VecDeque};
 use std::sync::{Arc, Weak};
 use std::path::{Path};
 use std::fmt;
+use std::cmp;
 use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering};
 use std::time::{Instant};
 use time::precise_time_ns;
@@ -73,6 +74,7 @@ pub use blockchain::CacheSize as BlockChainCacheSize;
 
 const MAX_TX_QUEUE_SIZE: usize = 4096;
 const MAX_QUEUE_SIZE_TO_SLEEP_ON: usize = 2;
+const MIN_HISTORY_SIZE: u64 = 8;
 
 impl fmt::Display for BlockChainInfo {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -139,10 +141,9 @@ pub struct Client {
 	notify: RwLock<Vec<Weak<ChainNotify>>>,
 	queue_transactions: AtomicUsize,
 	last_hashes: RwLock<VecDeque<H256>>,
+	history: u64,
 }
 
-const HISTORY: u64 = 1200;
-
 // database columns
 /// Column for State
 pub const DB_COL_STATE: Option<u32> = Some(0);
@@ -187,6 +188,19 @@ impl Client {
 			try!(db.write(batch).map_err(ClientError::Database));
 		}
 
+		trace!("Cleanup journal: DB Earliest = {:?}, Latest = {:?}", state_db.journal_db().earliest_era(), state_db.journal_db().latest_era());
+		let history = cmp::max(MIN_HISTORY_SIZE, config.history);
+		if let (Some(earliest), Some(latest)) = (state_db.journal_db().earliest_era(), state_db.journal_db().latest_era()) {
+			if latest > earliest && latest - earliest > history {
+				for era in earliest..(latest - history + 1) {
+					trace!("Removing era {}", era);
+					let batch = DBTransaction::new(&db);
+					try!(state_db.journal_db_mut().commit_old(&batch, era, &chain.block_hash(era).expect("Old block not found in the database")));
+					try!(db.write(batch).map_err(ClientError::Database));
+				}
+			}
+		}
+
 		if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(h.state_root())) {
 			warn!("State root not found for block #{} ({})", chain.best_block_number(), chain.best_block_hash().hex());
 		}
@@ -219,6 +233,7 @@ impl Client {
 			notify: RwLock::new(Vec::new()),
 			queue_transactions: AtomicUsize::new(0),
 			last_hashes: RwLock::new(VecDeque::new()),
+			history: history,
 		};
 		Ok(Arc::new(client))
 	}
@@ -275,7 +290,7 @@ impl Client {
 
 		// Check the block isn't so old we won't be able to enact it.
 		let best_block_number = self.chain.best_block_number();
-		if best_block_number >= HISTORY && header.number() <= best_block_number - HISTORY {
+		if best_block_number >= self.history && header.number() <= best_block_number - self.history {
 			warn!(target: "client", "Block import failed for #{} ({})\nBlock is ancient (current best block: #{}).", header.number(), header.hash(), best_block_number);
 			return Err(());
 		}
@@ -419,8 +434,8 @@ impl Client {
 		let number = block.header().number();
 		let parent = block.header().parent_hash().clone();
 		// Are we committing an era?
-		let ancient = if number >= HISTORY {
-			let n = number - HISTORY;
+		let ancient = if number >= self.history {
+			let n = number - self.history;
 			Some((n, self.chain.block_hash(n).expect("only verified blocks can be commited; verified block has hash; qed")))
 		} else {
 			None
@@ -498,7 +513,7 @@ impl Client {
 			let db = self.state_db.lock().boxed_clone();
 
 			// early exit for pruned blocks
-			if db.is_pruned() && self.chain.best_block_number() >= block_number + HISTORY {
+			if db.is_pruned() && self.chain.best_block_number() >= block_number + self.history {
 				return None;
 			}
 
@@ -603,7 +618,7 @@ impl Client {
 		let best_block_number = self.chain_info().best_block_number;
 		let block_number = try!(self.block_number(at).ok_or(snapshot::Error::InvalidStartingBlock(at)));
 
-		if best_block_number > HISTORY + block_number && db.is_pruned() {
+		if best_block_number > self.history + block_number && db.is_pruned() {
 			return Err(snapshot::Error::OldBlockPrunedDB.into());
 		}
 
@@ -615,8 +630,10 @@ impl Client {
 					0
 				};
 
-				self.block_hash(BlockID::Number(start_num))
-					.expect("blocks within HISTORY are always stored.")
+				match self.block_hash(BlockID::Number(start_num)) {
+					Some(h) => h,
+					None => return Err(snapshot::Error::InvalidStartingBlock(at).into()),
+				}
 			}
 			_ => match self.block_hash(at) {
 				Some(hash) => hash,

ethcore/src/client/config.rs

@@ -107,6 +107,8 @@ pub struct ClientConfig {
 	pub mode: Mode,
 	/// Type of block verifier used by client.
 	pub verifier_type: VerifierType,
+	/// State pruning history size.
+	pub history: u64,
 }
 
 #[cfg(test)]