Memory-based pruning history size (#4114)

* prune states based on memory param

* pruning memory CLI and usage in sync

* return purged value from memorydb

* calculate memory used incrementally in overlayrecentdb

* refactor shared history pruning code in client

* Fixed usage alignment

* journal_size function for fast memory calculation

ethcore/src/client/client.rs

@@ -206,17 +206,6 @@ impl Client {
 			config.history
 		};
 
-		if let (Some(earliest), Some(latest)) = (state_db.journal_db().earliest_era(), state_db.journal_db().latest_era()) {
-			if latest > earliest && latest - earliest > history {
-				for era in earliest..(latest - history + 1) {
-					trace!("Removing era {}", era);
-					let mut batch = DBTransaction::new(&db);
-					state_db.mark_canonical(&mut batch, era, &chain.block_hash(era).expect("Old block not found in the database"))?;
-					db.write(batch).map_err(ClientError::Database)?;
-				}
-			}
-		}
-
 		if !chain.block_header(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(h.state_root())) {
 			warn!("State root not found for block #{} ({})", chain.best_block_number(), chain.best_block_hash().hex());
 		}
@@ -257,6 +246,13 @@ impl Client {
 			on_mode_change: Mutex::new(None),
 			registrar: Mutex::new(None),
 		});
+
+		{
+			let state_db = client.state_db.lock().boxed_clone();
+			let chain = client.chain.read();
+			client.prune_ancient(state_db, &chain)?;
+		}
+
 		if let Some(reg_addr) = client.additional_params().get("registrar").and_then(|s| Address::from_str(s).ok()) {
 			trace!(target: "client", "Found registrar at {}", reg_addr);
 			let weak = Arc::downgrade(&client);
@@ -553,16 +549,6 @@ impl Client {
 		let mut state = block.drain();
 
 		state.journal_under(&mut batch, number, hash).expect("DB commit failed");
-
-		if number >= self.history {
-			let n = number - self.history;
-			if let Some(ancient_hash) = chain.block_hash(n) {
-				state.mark_canonical(&mut batch, n, &ancient_hash).expect("DB commit failed");
-			} else {
-				debug!(target: "client", "Missing expected hash for block {}", n);
-			}
-		}
-
 		let route = chain.insert_block(&mut batch, block_data, receipts);
 		self.tracedb.read().import(&mut batch, TraceImportRequest {
 			traces: traces.into(),
@@ -578,9 +564,49 @@ impl Client {
 		self.db.read().write_buffered(batch);
 		chain.commit();
 		self.update_last_hashes(&parent, hash);
+
+		if let Err(e) = self.prune_ancient(state, &chain) {
+			warn!("Failed to prune ancient state data: {}", e);
+		}
+
 		route
 	}
 
+	// prune ancient states until below the memory limit or only the minimum amount remain.
+	fn prune_ancient(&self, mut state_db: StateDB, chain: &BlockChain) -> Result<(), ClientError> {
+		let number = match state_db.journal_db().latest_era() {
+			Some(n) => n,
+			None => return Ok(()),
+		};
+
+		// prune all ancient eras until we're below the memory target,
+		// but have at least the minimum number of states.
+		loop {
+			let needs_pruning = state_db.journal_db().is_pruned() &&
+				state_db.journal_db().journal_size() >= self.config.history_mem;
+
+			if !needs_pruning { break }
+			match state_db.journal_db().earliest_era() {
+				Some(era) if era + self.history <= number => {
+					trace!(target: "client", "Pruning state for ancient era {}", era);
+					match chain.block_hash(era) {
+						Some(ancient_hash) => {
+							let mut batch = DBTransaction::new(&self.db.read());
+							state_db.mark_canonical(&mut batch, era, &ancient_hash)?;
+							self.db.read().write_buffered(batch);
+							state_db.journal_db().flush();
+						}
+						None =>
+							debug!(target: "client", "Missing expected hash for block {}", era),
+					}
+				}
+				_ => break, // means that every era is kept, no pruning necessary.
+			}
+		}
+
+		Ok(())
+	}
+
 	fn update_last_hashes(&self, parent: &H256, hash: &H256) {
 		let mut hashes = self.last_hashes.write();
 		if hashes.front().map_or(false, |h| h == parent) {
@@ -1408,7 +1434,6 @@ impl BlockChainClient for Client {
 		PruningInfo {
 			earliest_chain: self.chain.read().first_block_number().unwrap_or(1),
 			earliest_state: self.state_db.lock().journal_db().earliest_era().unwrap_or(0),
-			state_history_size: Some(self.history),
 		}
 	}
 

ethcore/src/client/config.rs

@@ -129,8 +129,10 @@ pub struct ClientConfig {
 	pub state_cache_size: usize,
 	/// EVM jump-tables cache size.
 	pub jump_table_size: usize,
-	/// State pruning history size.
+	/// Minimum state pruning history size.
 	pub history: u64,
+	/// Ideal memory usage for state pruning history.
+	pub history_mem: usize,
 	/// Check seal valididity on block import
 	pub check_seal: bool,
 }

ethcore/src/client/test_client.rs

@@ -714,10 +714,10 @@ impl BlockChainClient for TestBlockChainClient {
 	fn disable(&self) { unimplemented!(); }
 
 	fn pruning_info(&self) -> PruningInfo {
+		let best_num = self.chain_info().best_block_number;
 		PruningInfo {
 			earliest_chain: 1,
-			earliest_state: 1,
-			state_history_size: *self.history.read(),
+			earliest_state: self.history.read().as_ref().map(|x| best_num - x).unwrap_or(0),
 		}
 	}
 

ethcore/src/types/pruning_info.rs

@@ -28,6 +28,4 @@ pub struct PruningInfo {
 	pub earliest_chain: u64,
 	/// The first block where state requests may be served.
 	pub earliest_state: u64,
-	/// State pruning history size.
-	pub state_history_size: Option<u64>,
 }